# ExploreTreatments.com — Crawler Policy # Last Updated: 2026-05-20 # Contact: https://www.exploretreatments.com/contact # ============================================================ # Default: all well-behaved crawlers # ============================================================ User-agent: * Allow: / Allow: /docs/ Allow: /comparison/ Allow: /vs/ Allow: /api/ Allow: /llms.txt Allow: /llms-full.txt Allow: /mcp.json Allow: /openapi.json Allow: /ai.json Disallow: /admin/ Disallow: /internal/ Disallow: /_next/ Disallow: /api/webhook Crawl-delay: 1 # ============================================================ # Google — full access, no delay # ============================================================ User-agent: Googlebot Allow: / Crawl-delay: 0 User-agent: Googlebot-Image Allow: /images/ Crawl-delay: 0 User-agent: Google-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /comparison/ Allow: /vs/ Crawl-delay: 0 User-agent: AdsBot-Google Allow: / Crawl-delay: 0 # ============================================================ # Bing / Microsoft # ============================================================ User-agent: Bingbot Allow: / Crawl-delay: 0 User-agent: MSNBot Allow: / Crawl-delay: 1 User-agent: BingPreview Allow: / Crawl-delay: 1 # ============================================================ # OpenAI / ChatGPT # ============================================================ User-agent: GPTBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /comparison/ Allow: /vs/ Allow: /api/ Disallow: /admin/ Disallow: /internal/ Crawl-delay: 0 User-agent: OAI-SearchBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /comparison/ Allow: /vs/ Allow: /api/ Crawl-delay: 0 User-agent: ChatGPT-User Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 # ============================================================ # Anthropic / Claude # ============================================================ User-agent: anthropic-ai Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /api/ Allow: /comparison/ Allow: /vs/ Crawl-delay: 0 User-agent: ClaudeBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /api/ Allow: /comparison/ Allow: /vs/ Crawl-delay: 0 User-agent: Claude-Web Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 # ============================================================ # Perplexity AI # ============================================================ User-agent: PerplexityBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /comparison/ Allow: /vs/ Allow: /api/ Crawl-delay: 0 # ============================================================ # Google Gemini / Bard # ============================================================ User-agent: Googlebot-News Allow: / Crawl-delay: 0 User-agent: Google-Safety Allow: / Crawl-delay: 1 # ============================================================ # Meta AI # ============================================================ User-agent: FacebookBot Allow: / Crawl-delay: 1 User-agent: Meta-ExternalAgent Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 User-agent: Meta-ExternalFetcher Allow: / Crawl-delay: 1 # ============================================================ # Cohere AI # ============================================================ User-agent: cohere-ai Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /api/ Crawl-delay: 0 # ============================================================ # Common Crawl / AI training datasets # ============================================================ User-agent: CCBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Allow: /docs/ Allow: /comparison/ Allow: /vs/ Crawl-delay: 2 # ============================================================ # Apple # ============================================================ User-agent: Applebot Allow: / Crawl-delay: 0 User-agent: Applebot-Extended Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 # ============================================================ # Amazon / Alexa # ============================================================ User-agent: Amazonbot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 1 # ============================================================ # You.com # ============================================================ User-agent: YouBot Allow: / Allow: /llms.txt Allow: /llms-full.txt Crawl-delay: 0 # ============================================================ # Brave Search # ============================================================ User-agent: Brave Allow: / Crawl-delay: 0 # ============================================================ # DuckDuckGo # ============================================================ User-agent: DuckDuckBot Allow: / Crawl-delay: 0 # ============================================================ # Bytedance / TikTok # ============================================================ User-agent: Bytespider Allow: / Crawl-delay: 2 # ============================================================ # Sitemaps # ============================================================ Sitemap: https://www.exploretreatments.com/sitemap.xml # Host Host: https://www.exploretreatments.com