# CreatorDB — robots.txt
# All bots welcome, including AI/LLM crawlers used for citation in ChatGPT,
# Claude, Gemini, Perplexity, etc. Explicit Allows are documented for clarity
# even though "User-agent: *  Allow: /" already covers them.

User-agent: *
Allow: /
Disallow: /wp-admin/
Disallow: /wp-login.php

# Internal API endpoints — never meant to be indexed. /api/generate in
# particular was being crawled from JS-embedded refresh URLs, wasting crawl
# budget and triggering paid regenerations (force=1). Keep open only the two
# paths crawlers legitimately need: the creator sitemap, the avatar image
# proxy (Google Images), and the cheap recent-creators feed (internal links).
Disallow: /api/
Allow: /api/img-proxy
Allow: /api/sitemap-creators
Allow: /api/recent

# ------------------------------------------------------------------
# Explicit allows for major AI / LLM crawlers (citation traffic)
# ------------------------------------------------------------------

# OpenAI — ChatGPT browse + training crawler
User-agent: GPTBot
Allow: /

# OpenAI — used when ChatGPT browses URLs on a user's behalf
User-agent: ChatGPT-User
Allow: /

# OpenAI — search index crawler
User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude.ai citation crawler
User-agent: ClaudeBot
Allow: /

# Anthropic — historical name
User-agent: anthropic-ai
Allow: /

# Anthropic — Claude.ai web user agent
User-agent: Claude-Web
Allow: /

# Google — Gemini / SGE generative answers
User-agent: Google-Extended
Allow: /

# Perplexity — search and answers
User-agent: PerplexityBot
Allow: /

# Perplexity — user-on-demand browser
User-agent: Perplexity-User
Allow: /

# Common Crawl — corpus used by many models
User-agent: CCBot
Allow: /

# Apple Intelligence / Apple AI
User-agent: Applebot-Extended
Allow: /

# Bytedance / Doubao
User-agent: Bytespider
Allow: /

# Meta / Facebook AI
User-agent: Meta-ExternalAgent
Allow: /
User-agent: FacebookBot
Allow: /

# Diffbot — used by many enterprise AI products
User-agent: Diffbot
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Mistral
User-agent: MistralAI-User
Allow: /

# DuckDuckGo Assist
User-agent: DuckAssistBot
Allow: /

# ------------------------------------------------------------------
# Chinese search engines (for the /cn/ Simplified-Chinese pages)
# ------------------------------------------------------------------

# Baidu
User-agent: Baiduspider
Allow: /

# Sogou
User-agent: Sogou web spider
Allow: /

# 360 / Haosou
User-agent: 360Spider
Allow: /

# Shenma (UC / mobile)
User-agent: YisouSpider
Allow: /

# ------------------------------------------------------------------
# Sitemap + LLM discovery
# ------------------------------------------------------------------
Sitemap: https://creatordb.app/sitemap.xml
Sitemap: https://creatordb.app/sitemap-new-pages.xml
Sitemap: https://creatordb.app/sitemap-sponsorships.xml
Sitemap: https://creatordb.app/sitemap-brands.xml
Sitemap: https://creatordb.app/sitemap-creators-2.xml
Sitemap: https://creatordb.app/sitemap-cn.xml
Sitemap: https://creatordb.app/api/sitemap-creators/