User-agent: *
# contentsignals.org. Being quoted in an AI answer is a route to customers, so
# search and grounding are welcome; training a model on this copy returns
# nothing to us. Expresses a reservation of rights, and nothing more.
#
# search: build a search index and link back to the source.
# ai-input: read the page to answer a question now, citing it.
# ai-train: keep the text to train or fine-tune a model.
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /

# Nothing behind sign-in is useful to a crawler, and these paths carry
# single-use tokens and per-customer data.
# /auth.md is the exception: it is public documentation, and the Disallow below
# would otherwise block it on a prefix match.
Allow: /auth.md
Disallow: /app
Disallow: /auth
Disallow: /api

# Assistants are increasingly how people find a tool like this, so the ones that
# answer questions and cite a source are welcome. This is a courtesy to
# well-behaved crawlers and nothing more: it is advisory, unenforceable, and
# stops none of the scanners that forge these names.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

Sitemap: https://datanem.com/sitemap.xml