# LLMs.txt
# Policy for Large Language Model (LLM) crawlers
# Place this file at: https://yourdomain.com/llms.txt

# ================================
# DEFAULT POLICY
# ================================
# Allow all LLM crawlers access to public content.
User-agent: *
Allow: /

# Block sensitive or private sections
Disallow: /login/
Disallow: /checkout/
Disallow: /dashboard/
Disallow: /private/
Disallow: /members-only/
Disallow: /internal/
Disallow: /temp/

# ================================
# OPENAI - ChatGPT / GPTBot
# ================================
User-agent: GPTBot
Allow: /
Disallow: /login/
Disallow: /checkout/
Disallow: /dashboard/
Disallow: /private/

# ================================
# ANTHROPIC - ClaudeBot
# ================================
User-agent: ClaudeBot
Allow: /
Disallow: /login/
Disallow: /members-only/

# ================================
# PERPLEXITY.AI
# ================================
User-agent: PerplexityBot
Allow: /
Disallow: /internal/
Disallow: /temp/

# ================================
# GOOGLE AI CRAWLER
# ================================
User-agent: Google-Extended
# Controls how Google may use your content for Gemini / Bard training
Allow: /
Disallow: /sensitive/

# ================================
# META (Facebook / Instagram AI)
# ================================
User-agent: FacebookBot
Allow: /
Disallow: /private/
Disallow: /user-data/

# ================================
# APPLE AI
# ================================
User-agent: AppleBot
Allow: /
Disallow: /private/

# ================================
# COMMONCRAWL (used by many LLMs)
# ================================
User-agent: CCBot
Allow: /
Disallow: /sensitive/
Disallow: /private/

# ================================
# NOTES
# ================================
# 1. This configuration allows AI crawlers to use public content for
#    training and contextualisation (similar to SEO).
# 2. Sensitive areas (logins, dashboards, internal pages) are protected.
# 3. Not all AI companies honor llms.txt yet, but adoption is growing.