# damex.digital
# Search engines and AI assistants are welcome to crawl, read and cite this site.
#
# A crawler obeys the group (or groups, combined) that name it and ignores "*" once it has one
# (RFC 9309), so each group below repeats the same Disallow lines. Rules left only under "*"
# would not reach the bots named after it.
# Edit the three private paths in every group together (src/lib/robots-file.test.ts checks it).
#
# For people:   https://damex.digital/sitemap/
# For crawlers: https://damex.digital/sitemap-index.xml (also served at /sitemap.xml)
# For AI:       https://damex.digital/llms.txt (summary), /llms-full.txt (full text), and the full
#               text one section at a time at /llms/<section>.txt (listed in llms.txt).
#               Most pages also have a Markdown version at <path>.md, or send "Accept: text/markdown".
# Bot names below were checked on 2026-11-05. Review them every few months: vendors add and rename crawlers.

# Every other crawler, including Googlebot
User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=yes
Allow: /
Disallow: /api/
Disallow: /book/manage/
Disallow: /admin/

# AI search and answer engines: these decide whether Damex is named in an answer
User-agent: OAI-SearchBot
User-agent: PerplexityBot
User-agent: Claude-SearchBot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckAssistBot
User-agent: Amazonbot
Allow: /
Disallow: /api/
Disallow: /book/manage/
Disallow: /admin/

# AI assistants fetching a page because a person asked about it
User-agent: ChatGPT-User
User-agent: Perplexity-User
User-agent: Claude-User
User-agent: MistralAI-User
User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /api/
Disallow: /book/manage/
Disallow: /admin/

# AI training: crawlers that collect text to train models, plus control tokens
# (Google-Extended and Applebot-Extended don't crawl; they set whether content may be used for AI)
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: Meta-ExternalAgent
User-agent: CCBot
Allow: /
Disallow: /api/
Disallow: /book/manage/
Disallow: /admin/

Sitemap: https://damex.digital/sitemap-index.xml