# ============================================================
# robots.txt — Dr Nicolas Pellegrini — Chirurgien Orthopediste
# Site : https://www.dr-pellegrini-orthopedie.fr/
# Version : 2026-05-18 — GEO-ready (Generative Engine Optimization)
# Reference llms.txt : https://llmstxt.org/
# ============================================================

# ------------------------------------------------------------
# 1) Moteurs de recherche classiques
# ------------------------------------------------------------
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Qwantify
Allow: /

User-agent: YandexBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: WhatsApp
Allow: /

# ------------------------------------------------------------
# 2) BOTS IA — Crawlers de moteurs generatifs (GEO)
#    Politique : autorisation EXPLICITE sur le contenu public,
#    blocage des espaces admin/techniques.
#    Objectif : visibilite dans ChatGPT Search, Perplexity,
#    Claude, Google AI Overviews, Copilot, Brave Leo, etc.
# ------------------------------------------------------------

# --- OpenAI (ChatGPT) ---
# https://platform.openai.com/docs/bots
User-agent: GPTBot
Allow: /
Disallow: /admin/
Disallow: /strategy/
Disallow: /livrables-strategie/

User-agent: ChatGPT-User
Allow: /
Disallow: /admin/

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin/

# --- Anthropic (Claude) ---
# https://support.anthropic.com/en/articles/8896518
User-agent: ClaudeBot
Allow: /
Disallow: /admin/
Disallow: /strategy/

User-agent: Claude-Web
Allow: /
Disallow: /admin/

User-agent: Claude-SearchVerifier
Allow: /
Disallow: /admin/

User-agent: anthropic-ai
Allow: /
Disallow: /admin/

# --- Perplexity ---
# https://docs.perplexity.ai/guides/bots
User-agent: PerplexityBot
Allow: /
Disallow: /admin/

User-agent: Perplexity-User
Allow: /
Disallow: /admin/

# --- Google AI (Gemini, AI Overviews, Vertex) ---
# https://developers.google.com/search/docs/crawling-indexing/google-special-case-crawlers
User-agent: Google-Extended
Allow: /
Disallow: /admin/

User-agent: GoogleOther
Allow: /
Disallow: /admin/

User-agent: GoogleOther-Image
Allow: /
Disallow: /admin/

# --- Apple Intelligence ---
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /
Disallow: /admin/

# --- Microsoft (Copilot, Bing Chat) ---
User-agent: msnbot
Allow: /

User-agent: msnbot-media
Allow: /

# --- Meta AI (Llama, Meta-ExternalAgent) ---
User-agent: FacebookBot
Allow: /
Disallow: /admin/

User-agent: Meta-ExternalAgent
Allow: /
Disallow: /admin/

User-agent: Meta-ExternalFetcher
Allow: /
Disallow: /admin/

# --- Common Crawl (alimente de nombreux LLMs) ---
User-agent: CCBot
Allow: /
Disallow: /admin/

# --- You.com / Phind / Andi (moteurs IA) ---
User-agent: YouBot
Allow: /
Disallow: /admin/

User-agent: PhindBot
Allow: /
Disallow: /admin/

# --- Brave Leo / Brave Search ---
User-agent: Bravebot
Allow: /
Disallow: /admin/

# --- Cohere ---
User-agent: cohere-ai
Allow: /
Disallow: /admin/

User-agent: cohere-training-data-crawler
Allow: /
Disallow: /admin/

# --- Diffbot (source de donnees pour plusieurs LLMs) ---
User-agent: Diffbot
Allow: /
Disallow: /admin/

# --- ByteDance / TikTok / Doubao ---
User-agent: Bytespider
Allow: /
Disallow: /admin/

# --- Mistral AI ---
User-agent: MistralAI-User
Allow: /
Disallow: /admin/

# --- DuckAssist (DuckDuckGo AI) ---
User-agent: DuckAssistBot
Allow: /
Disallow: /admin/

# --- Kagi Assistant ---
User-agent: KagiBot
Allow: /
Disallow: /admin/

# --- Timpibot ---
User-agent: Timpibot
Allow: /
Disallow: /admin/

# ------------------------------------------------------------
# 3) Politique generique pour tous les autres bots
#    NB : pas de Crawl-delay agressif (nuit au GEO real-time)
# ------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /cgi-bin/
Disallow: /.htaccess
Disallow: /.htpasswd
Disallow: /.git/
Disallow: /tmp/
Disallow: /_archive/
Disallow: /_obsidian-export/
Disallow: /node_modules/
Disallow: /strategy/
Disallow: /livrables-strategie/

# ------------------------------------------------------------
# 4) Index des ressources structurees
# ------------------------------------------------------------
Sitemap: https://www.dr-pellegrini-orthopedie.fr/sitemap.xml

# Ressources LLM-friendly (proposition llmstxt.org)
# https://www.dr-pellegrini-orthopedie.fr/llms.txt
# https://www.dr-pellegrini-orthopedie.fr/llms-full.txt