# By default, all web crawlers are denied access to non-view actions and UI resources (images, js, css)
User-agent: *

Disallow: */viewattachrev/
Disallow: */viewrev/
Disallow: */pdf/
Disallow: */tex/
Disallow: */edit/
Disallow: */create/
Disallow: */inline/
Disallow: */preview/
Disallow: */save/
Disallow: */saveandcontinue/
Disallow: */rollback/
Disallow: */deleteversions/
Disallow: */cancel/
Disallow: */delete/
Disallow: */deletespace/
Disallow: */undelete/
Disallow: */reset/
Disallow: */register/
Disallow: */propupdate/
Disallow: */propadd/
Disallow: */propdisable/
Disallow: */propenable/
Disallow: */propdelete/
Disallow: */objectadd/
Disallow: */commentadd/
Disallow: */commentsave/
Disallow: */objectsync/
Disallow: */objectremove/
Disallow: */attach/
Disallow: */upload/
Disallow: */download/
Disallow: */temp/
Disallow: */downloadrev/
Disallow: */dot/
Disallow: */svg/
Disallow: */delattachment/
Disallow: */skin/
Disallow: */jsx/
Disallow: */ssx/
Disallow: */login/
Disallow: */loginsubmit/
Disallow: */loginerror/
Disallow: */logout/
Disallow: */charting/
Disallow: */lock/
Disallow: */redirect/
Disallow: */admin/
Disallow: */export/
Disallow: */import/
Disallow: */get/
Disallow: */distribution/
Disallow: */imagecaptcha/
Disallow: */unknown/
Disallow: */webjars/
Disallow: */resources/
# Well known application (non-content) locations.
Disallow: */Sandbox/
Disallow: */Admin/
Disallow: */Stats/
Disallow: */Panels/
# We're not interested in rendering and indexing panels.
Disallow: */asyncrenderer/uix/*Panel*
Disallow: */Main/Search
# XWiki virtual users that do not have profile pages. Avoid unnecessary 404 requests/errors.
Disallow: */XWiki/XWikiGuest
Disallow: */XWiki/superadmin
# Avoid crawling unnecesary UI elements that are not relevant for indexing and can even cause loops (like pdfoptions, etc.)
Disallow: /*?*xpage=*
# Index only the main page content, all other viewers are not relevant for idexing.
Disallow: */view/*?*viewer=*
# Don't index the REST API.
Disallow: */rest/

# For images uploaded as attachments inside wiki pages
Allow: */*/download/*.png$
Allow: */*/download/*.jpg$
Allow: */*/download/*.jpeg$
Allow: */*/download/*.gif$

# Googlebot uses a headless browser to fully render a page before indexing, so the UI resources are relevant and actually needed.
User-agent: Googlebot
# JS
Allow: */jsx/
Allow: */get/*?*xpage=plain*
Allow: */webjars/
# CSS
Allow: */ssx/
# Images
Allow: */charting/
Allow: */dot/
Allow: */svg/
Allow: */download/*.png
Allow: */download/*.jpg
Allow: */download/*.jpeg
Allow: */download/*.gif
Allow: */download/*.svg
# A bit of everything
Allow: */skin/
Allow: */resources/
# We're not interested in rendering and indexing panels.
Disallow: */asyncrenderer/uix/*Panel*
Disallow: */jsx/*Panel*
Disallow: */ssx/*Panel*
# All other rules stay the same for Googlebot. We seem to have to add them or they will default to Allow and not inherit from the generic rules.
Disallow: */viewattachrev/
Disallow: */viewrev/
Disallow: */pdf/
Disallow: */tex/
Disallow: */edit/
Disallow: */create/
Disallow: */inline/
Disallow: */preview/
Disallow: */save/
Disallow: */saveandcontinue/
Disallow: */rollback/
Disallow: */deleteversions/
Disallow: */cancel/
Disallow: */delete/
Disallow: */deletespace/
Disallow: */undelete/
Disallow: */reset/
Disallow: */register/
Disallow: */propupdate/
Disallow: */propadd/
Disallow: */propdisable/
Disallow: */propenable/
Disallow: */propdelete/
Disallow: */objectadd/
Disallow: */commentadd/
Disallow: */commentsave/
Disallow: */objectsync/
Disallow: */objectremove/
Disallow: */attach/
Disallow: */upload/
Disallow: */download/
Disallow: */temp/
Disallow: */downloadrev/
Disallow: */delattachment/
Disallow: */login/
Disallow: */loginsubmit/
Disallow: */loginerror/
Disallow: */logout/
Disallow: */lock/
Disallow: */redirect/
Disallow: */admin/
Disallow: */export/
Disallow: */import/
Disallow: */get/
Disallow: */distribution/
Disallow: */imagecaptcha/
Disallow: */unknown/
# Well known application (non-content) locations.
Disallow: */Sandbox/
Disallow: */Admin/
Disallow: */Stats/
Disallow: */Panels/
Disallow: */Main/Search
# XWiki virtual users that do not have profile pages. Avoid unnecessary 404 requests/errors.
Disallow: */XWiki/XWikiGuest
Disallow: */XWiki/superadmin
# Avoid crawling unnecesary UI elements that are not relevant for indexing and can even cause loops (like pdfoptions, etc.)
Disallow: /*?*xpage=*
# Index only the main page content, all other viewers are not relevant for idexing.
Disallow: /*?*viewer=*
# Don't index the REST API.
Disallow: */rest/

# For images uploaded as attachments inside wiki pages
Allow: */*/download/*.png$
Allow: */*/download/*.jpg$
Allow: */*/download/*.jpeg$
Allow: */*/download/*.gif$

# Block ChatGPT
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
# Block Google AI
User-agent: Google-Extended
Disallow: /
# Block PerplexityBot
User-agent: PerplexityBot
Disallow: /
# Block Anthropic AI
User-agent: anthropic-ai
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: ClaudeBot
Disallow: /