#####################################################################
# robots.txt for ucalyptus.me
#
# Policy: Optimized for Answer Engine Optimization (AEO).
# Legit AI answer / retrieval bots are ALLOWED so ucalyptus.me can be
# cited by ChatGPT Search, Claude, Perplexity, Google Gemini,
# Bing Copilot, Apple Intelligence, Meta AI, Amazon Alexa, Mistral,
# Cohere, DuckAssist, Kagi, You.com, Brave, and open-research crawlers.
# Aggressive / non-consent / spam scrapers are BLOCKED.
#
# Content-Signal declares the same "come here" policy in the
# machine-readable format at https://contentsignals.org/ — this site
# wants to be trained on, searched, and used as AI answer input.
#
# See also:
#   Sitemap:  https://ucalyptus.me/sitemap.xml
#   llms.txt: https://ucalyptus.me/llms.txt
#   Agent:    https://ucalyptus.me/.well-known/agent.json
#
# Canonical AI-crawler taxonomy: https://github.com/ai-robots-txt/ai.robots.txt
#####################################################################


# NLWeb Schema Feeds: point crawlers at machine-readable structured-data feeds.
Schemamap: /schema-map.xml
# ---------- Default: allow all ----------
User-agent: *
Allow: /
Content-Signal: ai-train=yes, search=yes, ai-input=yes

# ---------- OpenAI (ChatGPT Search, ChatGPT, Operator, Agent) ----------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: ChatGPT Agent
Allow: /

User-agent: Operator
Allow: /

User-agent: NovaAct
Allow: /

# ---------- Anthropic (Claude Search, Claude, Claude Code) ----------
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-Code
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

# ---------- Perplexity ----------
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# ---------- Google (Search, Gemini, NotebookLM, Deep Research, Mariner) ----------
User-agent: Googlebot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: Google-CloudVertexBot
Allow: /

User-agent: Google-NotebookLM
Allow: /

User-agent: Gemini-Deep-Research
Allow: /

User-agent: Google-Gemini-CLI
Allow: /

User-agent: Google-Firebase
Allow: /

User-agent: GoogleAgent-Mariner
Allow: /

User-agent: GoogleAgent-URLContext
Allow: /

User-agent: Google-Agent
Allow: /

# ---------- Microsoft (Bing, Copilot, Azure) ----------
User-agent: Bingbot
Allow: /

User-agent: AzureAI-SearchBot
Allow: /

# ---------- Apple (Siri, Apple Intelligence) ----------
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ---------- Amazon (Alexa, Q Business, Bedrock, Buy For Me) ----------
User-agent: Amazonbot
Allow: /

User-agent: Amzn-User
Allow: /

User-agent: Amzn-SearchBot
Allow: /

User-agent: AmazonBuyForMe
Allow: /

User-agent: bedrockbot
Allow: /

# ---------- Meta (Meta AI) ----------
User-agent: Meta-ExternalAgent
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

User-agent: meta-externalfetcher
Allow: /

User-agent: meta-webindexer
Allow: /

User-agent: FacebookBot
Allow: /

# ---------- Cohere ----------
User-agent: cohere-ai
Allow: /

User-agent: cohere-training-data-crawler
Allow: /

# ---------- Mistral ----------
User-agent: MistralAI-User
Allow: /

# ---------- DuckDuckGo (DuckAssist) ----------
User-agent: DuckAssistBot
Allow: /

# ---------- You.com ----------
User-agent: YouBot
Allow: /

# ---------- Kagi ----------
User-agent: kagi-fetcher
Allow: /

# ---------- Brave Search AI ----------
User-agent: Bravebot
Allow: /

# ---------- Diffbot (structured extraction) ----------
User-agent: Diffbot
Allow: /

# ---------- Allen Institute (Ai2 open research) ----------
User-agent: AI2Bot
Allow: /

User-agent: Ai2Bot-Dolma
Allow: /

User-agent: AI2Bot-DeepResearchEval
Allow: /

# ---------- Exa (exa.ai / metaphor) ----------
User-agent: ExaBot
Allow: /

# ---------- Tavily ----------
User-agent: TavilyBot
Allow: /

# ---------- Firecrawl (fetch on demand) ----------
User-agent: FirecrawlAgent
Allow: /

# ---------- Phind ----------
User-agent: PhindBot
Allow: /

# ---------- Moonshot (Kimi) ----------
User-agent: Kimi-User
Allow: /

# ---------- Linkup ----------
User-agent: LinkupBot
Allow: /

# ---------- iAsk ----------
User-agent: iAskBot
Allow: /

# ---------- bigsur.ai ----------
User-agent: bigsur.ai
Allow: /

# ---------- Andibot (Andi search) ----------
User-agent: Andibot
Allow: /

# ---------- AIWebIndex (open web index for AI) ----------
User-agent: AIWebIndex
Allow: /

# ---------- Manus, Devin, Cursor (agentic IDEs / dev agents) ----------
User-agent: Manus-User
Allow: /

User-agent: Devin
Allow: /

User-agent: Cursor
Allow: /

User-agent: opencode
Allow: /

User-agent: Code
Allow: /

User-agent: Trae
Allow: /

# ---------- Cloudflare AutoRAG ----------
User-agent: Cloudflare-AutoRAG
Allow: /

# ---------- Other agentic retrievers ----------
User-agent: BuddyBot
Allow: /

User-agent: Crawl4AI
Allow: /

User-agent: Webzio-Extended
Allow: /

User-agent: webzio-extended
Allow: /

# ---------- Blocked: aggressive / non-consent / spam scrapers ----------
User-agent: Bytespider
Disallow: /

User-agent: TikTokSpider
Disallow: /

User-agent: TikTok Spider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Baidu
Disallow: /

User-agent: iaskspider
Disallow: /

User-agent: iaskspider/2.0
Disallow: /

User-agent: ChatGLM-Spider
Disallow: /

User-agent: YandexAdditional
Disallow: /

User-agent: YandexAdditionalBot
Disallow: /

User-agent: SemrushBot-OCOB
Disallow: /

User-agent: SemrushBot-SWA
Disallow: /

User-agent: PanguBot
Disallow: /

User-agent: TongyiBot
Disallow: /

User-agent: DeepSeekBot
Disallow: /

Sitemap: https://ucalyptus.me/sitemap.xml