# HexaHQ marketing site (www.hexahq.ai).
# Keep this maintained as pages are added/removed; see www/CLAUDE.md.
#
# EVERY CRAWLER IS ALLOWED, AND THE AI CRAWLERS ARE NAMED EXPLICITLY ON PURPOSE.
# The wildcard below already permits them, so the named blocks change no behavior today.
# They are here as a standing declaration, because the default for two of these tokens is
# not what it looks like: Google-Extended and Applebot-Extended do nothing for ordinary
# search indexing and exist ONLY to opt a site OUT of AI grounding and training. A site
# picks them up by accident, from a copied robots.txt or a "block the AI scrapers" pass,
# and the loss is invisible: nothing 404s, traffic looks normal, the site simply stops
# being quotable by the assistants. Naming each one Allow makes removing it a deliberate
# act with a visible diff rather than a default someone inherits.
#
# This is a deliberate business decision, not an oversight. HexaHQ is a product people find
# by asking an assistant how to govern what their AI can do; being readable by those
# assistants is a distribution channel, not a leak. Everything here is public marketing
# copy. If that trade ever changes, change it here, and say why.

User-agent: *
Allow: /
# SSI fragments, not standalone pages:
Disallow: /partials/
# Utility page rendered to the OG image; not a real page:
Disallow: /og-template.html
Disallow: /article-og-template.html

# --- Search indexing ---
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

# --- AI answer engines and assistant retrieval ---
# OpenAI: GPTBot trains, OAI-SearchBot builds the ChatGPT search index, ChatGPT-User
# fetches a page live when someone asks about it in a conversation.
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Anthropic: ClaudeBot trains, Claude-SearchBot indexes for Claude's search, Claude-User
# fetches on behalf of someone in a conversation.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google's AI grounding and training token. Absent = allowed, but see the note above.
User-agent: Google-Extended
Allow: /

# Apple Intelligence / Siri grounding. Same opt-out-only semantics as Google-Extended.
User-agent: Applebot-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: meta-externalagent
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

# Common Crawl. Not an assistant itself, but a corpus most of them have read.
User-agent: CCBot
Allow: /

Sitemap: https://www.hexahq.ai/sitemap.xml