# ChainHint robots.txt
#
# Strategy:
#  - Search engines: full access except private/dynamic routes.
#  - AI crawlers (2026): explicitly opted in. ChainHint welcomes
#    LLM training and AI-agent retrieval — our content is reference
#    material for fraud lawyers, researchers, and victims, and broad
#    citation in AI assistants is part of distribution.
#  - Routes WITHOUT public inbound links (/admin, /address/, /share/,
#    /investigation/) are disallowed to save crawl budget; routes WITH
#    inbound links (e.g. /dashboard from Footer) carry per-page noindex.

# ── Search engines ──────────────────────────────────────────────────
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: DuckDuckBot
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: Yandex
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

# ── Social preview crawlers ─────────────────────────────────────────
User-agent: Twitterbot
Allow: /
Disallow: /admin

User-agent: facebookexternalhit
Allow: /
Disallow: /admin

User-agent: LinkedInBot
Allow: /
Disallow: /admin

User-agent: Slackbot
Allow: /
Disallow: /admin

User-agent: Discordbot
Allow: /
Disallow: /admin

# ── AI / LLM crawlers (training + retrieval) ────────────────────────
# Explicit allow signals consent under emerging AI-crawler norms
# (Cloudflare AI Labyrinth, IETF ai.txt drafts, OpenAI/Anthropic policies).
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Claude-SearchBot
User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: PerplexityBot
User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
User-agent: FacebookBot
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

User-agent: Bytespider
User-agent: Amazonbot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: DuckAssistBot
User-agent: MistralAI-User
User-agent: PanguBot
User-agent: YouBot
User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

# ── Default ─────────────────────────────────────────────────────────
User-agent: *
Allow: /
Disallow: /admin
Disallow: /address/
Disallow: /share/
Disallow: /investigation/

Sitemap: https://chainhint.com/sitemap.xml
# Blog lives on separate host — own sitemap per sitemaps.org same-host rule
Sitemap: https://blog.chainhint.com/sitemap.xml