# robots.txt — AI Readiness Reference
# Example Analytics | https://exampleanalytics.com
# Generated: 2026-06-16
#
# IMPORTANT: This is a REFERENCE file, not a replacement for your live robots.txt.
# Merge the AI crawler blocks below into your existing robots.txt.
# Do NOT overwrite your current file — add these blocks to it.

# ─────────────────────────────────────────────
# STANDARD CRAWLERS (merge with existing rules)
# ─────────────────────────────────────────────

# Content-Signal (Cloudflare Content Signals Policy): tune yes/no per your
# training-data-policy.txt — search=indexing, ai-input=RAG/answers, ai-train=model training.

User-agent: *
Content-Signal: search=yes, ai-input=yes, ai-train=no
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /checkout/
Disallow: /billing/
Crawl-delay: 5

# ─────────────────────────────────────────────
# AI CRAWLERS — Add these blocks to your robots.txt
# ─────────────────────────────────────────────

# OpenAI / ChatGPT
User-agent: GPTBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /ai-sitemap.xml
Allow: /rag-index.json
Allow: /rag-index.jsonl
Allow: /ai-entities.json
Allow: /ai-intent.json
Allow: /ai-schema.json
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /billing/

# Anthropic / Claude
User-agent: ClaudeBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /ai-sitemap.xml
Allow: /rag-index.json
Allow: /rag-index.jsonl
Allow: /ai-entities.json
Allow: /ai-intent.json
Allow: /ai-schema.json
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /billing/

# Google AI / Gemini
User-agent: Google-Extended
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /ai-sitemap.xml
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /billing/

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /ai.txt
Allow: /ai-sitemap.xml
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /billing/

# Meta AI
User-agent: FacebookBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /billing/

# Common Crawl (used by many AI training datasets)
User-agent: CCBot
Allow: /
Disallow: /app/
Disallow: /api/
Disallow: /admin/
Disallow: /billing/
Crawl-delay: 10

# Cohere AI
User-agent: cohere-ai
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /app/
Disallow: /api/

# Diffbot
User-agent: Diffbot
Allow: /
Disallow: /app/
Disallow: /api/

# ─────────────────────────────────────────────
# SITEMAPS — Add these lines to your robots.txt
# ─────────────────────────────────────────────

Sitemap: https://exampleanalytics.com/sitemap.xml
Sitemap: https://exampleanalytics.com/ai-sitemap.xml

# ─────────────────────────────────────────────
# AI READINESS FILES (for reference — all publicly accessible)
# ─────────────────────────────────────────────
# https://exampleanalytics.com/ai.txt
# https://exampleanalytics.com/llms.txt
# https://exampleanalytics.com/llms-full.txt
# https://exampleanalytics.com/ai-sitemap.xml
# https://exampleanalytics.com/sitemap.md
# https://exampleanalytics.com/ai-entities.json
# https://exampleanalytics.com/ai-intent.json
# https://exampleanalytics.com/ai-schema.json
# https://exampleanalytics.com/rag-index.json
# https://exampleanalytics.com/rag-index.jsonl
# https://exampleanalytics.com/ai-disclosure.txt
# https://exampleanalytics.com/training-data-policy.txt
# https://exampleanalytics.com/.well-known/ai-plugin.json
# https://exampleanalytics.com/manifest.json