# ==============================================================================
# ROBOTS.TXT FOR MAGICLIGHT AI (https://en.magiclight.app)
# Standard compliant with RFC 9309 (Google, Bing, Yahoo, Yandex, DuckDuckGo)
# Specially optimized for Generative AI Bots (OpenAI, Anthropic, Perplexity, Apple, Meta)
# ==============================================================================

# ------------------------------------------------------------------------------
# 1. SEARCH ENGINES (Standard Search Engine Crawlers)
# ------------------------------------------------------------------------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Googlebot-Video
User-agent: Googlebot-News
Allow: /
Disallow: /api/
Disallow: /star

User-agent: Bingbot
User-agent: msnbot
Allow: /
Disallow: /api/
Disallow: /star

User-agent: DuckDuckBot
User-agent: YandexBot
User-agent: Baiduspider
User-agent: Slurp
User-agent: Sogou
Allow: /
Disallow: /api/
Disallow: /star

# ------------------------------------------------------------------------------
# 2. GENERATIVE AI & LLM RETRIEVAL BOTS (GEO & SearchGPT / Claude / Perplexity)
# ------------------------------------------------------------------------------
# OpenAI (SearchGPT, ChatGPT Web Browsing, Model Training)
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
Allow: /

# Google AI & Gemini Deep Research
User-agent: Google-Extended
Allow: /

# Anthropic Claude & Claude Search
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /

# Perplexity AI Search Engine
User-agent: PerplexityBot
Allow: /

# Apple Intelligence
User-agent: Applebot
User-agent: Applebot-Extended
Allow: /

# Meta AI / Llama
User-agent: Meta-ExternalAgent
User-agent: Meta-ExternalFetcher
Allow: /

# Common Crawl & Open Web Datasets for LLMs
User-agent: CCBot
User-agent: Diffbot
User-agent: Bytespider
User-agent: cohere-ai
Allow: /

# ------------------------------------------------------------------------------
# 3. GLOBAL FALLBACK
# ------------------------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /api/
Disallow: /star

# ------------------------------------------------------------------------------
# 4. SITEMAP & KNOWLEDGE BASE FOR AI AGENTS
# ------------------------------------------------------------------------------
Sitemap: https://en.magiclight.app/sitemap.xml