# Content Signals (contentsignals.org) declare what this site permits by
# category, separately from whether a crawler may fetch a page at all.
#
#   search=yes    index it and link to it, including short excerpts
#   ai-input=yes  use it to ground an answer, i.e. retrieval-augmented
#                 generation. This is the one that matters here: it is the
#                 path by which an assistant reads a page and cites it,
#                 and it is where this site's referral traffic comes from.
#                 The example in most write-ups sets ai-input=no, which
#                 would switch that off. Deliberately yes.
#   ai-train=no   do not use it as model training data. Training returns
#                 nothing to the site; grounding returns a citation.
#
# Any restriction here is an express reservation of rights under Article 4
# of EU Directive 2019/790.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /api/
Disallow: /health
Disallow: /docs
Disallow: /openapi.json
Allow: /openapi-public.json
Allow: /.well-known/
# The MCP endpoint and the page describing it. /mcp answers a plain GET with
# its tool spec and needs no key, so a directory or a client can inspect what
# it offers before connecting.
Allow: /mcp
Allow: /mcp-server
Disallow: /account
Disallow: /welcome
Disallow: /welcome-free
Disallow: /invite

# Explicit allow for AI answer-engine crawlers. Some of these obey
# robots but need their own user-agent line to be treated as first-
# class citizens vs generic `User-agent: *`. Listing them here is a
# signal we want them to index our content for use in AI responses.
User-agent: GPTBot
Allow: /
Disallow: /api/
Disallow: /account
Disallow: /welcome
Disallow: /welcome-free
Disallow: /invite

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /account

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /
Disallow: /api/
Disallow: /account

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /account

User-agent: Claude-User
Allow: /

# Google-Extended governs Gemini grounding and training only. It does
# NOT control AI Overviews or AI Mode, which follow Googlebot's rules.
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Bytespider
Allow: /
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /api/

# Newer answer-engine / assistant crawlers. These already fall under the
# permissive "User-agent: *" default; naming them explicitly is belt-and-
# suspenders so a citation-relevant bot is never accidentally caught by a
# future restrictive default.
User-agent: Meta-ExternalAgent
Allow: /
Disallow: /api/

User-agent: Amazonbot
Allow: /
Disallow: /api/

User-agent: DuckAssistBot
Allow: /
Disallow: /api/

User-agent: cohere-ai
Allow: /
Disallow: /api/

# Sitemap. Note that /llms.txt is published for agent tooling rather
# than for crawlers: measured adoption studies find answer engines do not
# request it, so it is not part of how this site gets cited.
Sitemap: https://snagres.com/sitemap.xml

# Machine-readable entry points, also advertised via Link headers on every
# HTML response: /.well-known/api-catalog, /openapi-public.json, /.well-known/agent.md