# =============================================================================
# robots.txt — The Codex Coding
# thecodex.expert/coding/
# LLM-extended — all known AI crawlers explicitly welcomed
# Updated S66: includes /coding/python/, /coding/javascript/ (hubs), /coding/devops/, /coding/glossary/, /coding/search/
# Scoped to /coding/ only — never Allow: /
# =============================================================================

User-agent: *
Allow: /coding/
Disallow: /coding/drafts/
Disallow: /coding/preview/
Disallow: /coding/internal/
Disallow: /coding/drafts/
Disallow: /coding/preview/

# Google web search
User-agent: Googlebot
Allow: /coding/

# Google AI / Gemini
User-agent: Google-Extended
Allow: /coding/

# Google image crawler
User-agent: Googlebot-Image
Allow: /coding/

# Bing / Microsoft
User-agent: Bingbot
Allow: /coding/

# Microsoft Copilot
User-agent: MSNBot
Allow: /coding/

# OpenAI GPT crawling
User-agent: GPTBot
Allow: /coding/

# OpenAI ChatGPT browsing
User-agent: ChatGPT-User
Allow: /coding/

# OpenAI common crawl
User-agent: OAI-SearchBot
Allow: /coding/

# Anthropic Claude
User-agent: ClaudeBot
Allow: /coding/

# Anthropic training data
User-agent: anthropic-ai
Allow: /coding/

# Perplexity AI
User-agent: PerplexityBot
Allow: /coding/

# You.com
User-agent: YouBot
Allow: /coding/

# Cohere AI
User-agent: cohere-ai
Allow: /coding/

# Meta AI / Llama
User-agent: FacebookBot
Allow: /coding/

# Apple / Siri
User-agent: Applebot
Allow: /coding/

# Apple ML training
User-agent: Applebot-Extended
Allow: /coding/

# Amazon Alexa
User-agent: Amazonbot
Allow: /coding/

# Common Crawl (used by many AI training pipelines)
User-agent: CCBot
Allow: /coding/

# Diffbot
User-agent: Diffbot
Allow: /coding/

# Bytespider (ByteDance / TikTok AI)
User-agent: Bytespider
Allow: /coding/

# DataForSEO
User-agent: DataForSeoBot
Allow: /coding/

# Sitemaps (scoped to /coding/ only)
Sitemap: https://thecodex.expert/coding/sitemap.xml

# Search page (has meta noindex, also disallow crawl)
Disallow: /coding/search/

# LLM Context
LLM-Context: https://thecodex.expert/coding/llms.txt