User-agent: *
Disallow: /cdn-cgi/

# OpenAI (ChatGPT, GPTBot)
User-agent: GPTBot
Disallow: /cdn-cgi/

User-agent: ChatGPT-User
Disallow: /cdn-cgi/

# Anthropic (Claude)
User-agent: ClaudeBot
Disallow: /cdn-cgi/

# Google AI (Gemini, Bard)
User-agent: Google-Extended
Disallow: /cdn-cgi/

# Perplexity AI
User-agent: PerplexityBot
Disallow: /cdn-cgi/

# xAI / Grok (Twitter/X) — emerging agents
User-agent: xAIBot
Disallow: /cdn-cgi/

User-agent: GrokBot
Disallow: /cdn-cgi/

User-agent: Twitterbot
Disallow: /cdn-cgi/

# Common Crawl (used by many LLMs)
User-agent: CCBot
Disallow: /cdn-cgi/

# Amazon (used in AI/ML pipelines)
User-agent: Amazonbot
Disallow: /cdn-cgi/

# Facebook / Meta AI
User-agent: facebookexternalhit
Disallow: /cdn-cgi/

User-agent: Facebot
Disallow: /cdn-cgi/

# DuckDuckGo AI
User-agent: DuckDuckBot
Disallow: /cdn-cgi/

# Neeva (now part of Snowflake)
User-agent: NeevaBot
Disallow: /cdn-cgi/

# You.com
User-agent: YouBot
Disallow: /cdn-cgi/

# Andi AI
User-agent: andi-bot
Disallow: /cdn-cgi/

# Phind (dev AI search engine)
User-agent: PhindBot
Disallow: /cdn-cgi/

# Bing / Microsoft AI
User-agent: bingbot
Disallow: /cdn-cgi/

User-agent: BingPreview
Disallow: /cdn-cgi/

# Yandex AI / General crawler
User-agent: YandexBot
Disallow: /cdn-cgi/

# Apple AI systems (Siri, Spotlight)
User-agent: Applebot
Disallow: /cdn-cgi/

# AhrefsBot (used in AI/data SEO)
User-agent: AhrefsBot
Disallow: /cdn-cgi/

Sitemap: https://southeastdetoxga.com/sitemap_index.xml