# public/robots.txt
# Top Tier Presence (https://toptierpresence.com)
# AI crawler welcome block: every agent listed below may fetch every public URL on this site.
# Machine manifests: https://toptierpresence.com/llms.txt and https://toptierpresence.com/llms-full.txt
# Updated: 2026-10-13

User-agent: OAI-SearchBot          # OpenAI: ChatGPT search index
User-agent: ChatGPT-User           # OpenAI: user-initiated fetches inside ChatGPT
User-agent: GPTBot                 # OpenAI: model training
User-agent: Claude-SearchBot       # Anthropic: Claude search index
User-agent: Claude-User            # Anthropic: user-initiated fetches inside Claude
User-agent: ClaudeBot              # Anthropic: model training
User-agent: Claude-Web             # Anthropic: legacy token, kept for older parsers
User-agent: anthropic-ai           # Anthropic: legacy token, kept for older parsers
User-agent: PerplexityBot          # Perplexity: search index
User-agent: Perplexity-User        # Perplexity: user-initiated fetches
User-agent: Google-Extended        # Google: Gemini training and grounding (robots token, not a separate crawler)
User-agent: Googlebot              # Google: Search, AI Overviews, AI Mode
User-agent: Bingbot                # Microsoft: Bing index and Copilot
User-agent: Applebot               # Apple: Siri and Spotlight
User-agent: Applebot-Extended      # Apple: Apple Intelligence training (robots token)
User-agent: Meta-ExternalAgent     # Meta: Meta AI training and retrieval
User-agent: Meta-ExternalFetcher   # Meta: user-initiated fetches
User-agent: FacebookBot            # Meta: language model improvement
User-agent: xAI-Grok               # xAI: observed Grok token (xAI has not published crawler docs)
User-agent: GrokBot                # xAI: observed Grok token
User-agent: Grok-DeepSearch        # xAI: observed Grok DeepSearch token
User-agent: DuckAssistBot          # DuckDuckGo: DuckAssist answers
User-agent: Amazonbot              # Amazon: Alexa and Amazon services
User-agent: MistralAI-User         # Mistral: Le Chat user-initiated fetches
User-agent: CCBot                  # Common Crawl: open web corpus used by many model builders
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# DeepSeek publishes no crawler token; it and every other agent fall under this group.
User-agent: *
Allow: /
Content-Signal: search=yes, ai-input=yes, ai-train=yes

Sitemap: https://toptierpresence.com/sitemap-index.xml