# Dugout Edge - robots.txt
# https://www.dugoutedge.com/robots.txt

# Allow all user agents to crawl the site
User-agent: *
Allow: /

# ===========================================
# AI/LLM Crawlers - Explicitly Allowed
# ===========================================

# OpenAI GPT crawlers
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic Claude crawlers
User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Common Crawl (used by many AI systems)
User-agent: CCBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

# Microsoft/Bing AI
User-agent: bingbot
Allow: /

# ===========================================
# Standard Search Engine Crawlers
# ===========================================

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# ===========================================
# Disallowed Paths
# ===========================================

# Protect user-specific API routes
User-agent: *
Disallow: /api/user/
Disallow: /api/stripe/
Disallow: /api/webhooks/
Disallow: /api/subscription/

# Allow public API routes for AI discovery
User-agent: *
Allow: /api/manifest
Allow: /api/drills
Allow: /api/glossary
Allow: /api/handbook
Allow: /api/articles
Allow: /api/openapi.json

# ===========================================
# Sitemaps
# ===========================================

Sitemap: https://www.dugoutedge.com/sitemap.xml

# ===========================================
# LLM-specific resources
# ===========================================

# AI agents should read this for site capabilities
# LLMs-txt: https://www.dugoutedge.com/llms.txt