User-agent: *
Allow: /
Disallow: /api/
# Block the Next.js build output EXCEPT the static assets a renderer needs.
# Bing was logging hundreds of robots violations fetching /_next/static/chunks/*
# — it has to execute the page JS and load the CSS to render, and a blanket
# Disallow: /_next/ told it not to. A crawler that cannot render sees an empty
# page. /_next/static/ is content-hashed public output; there is nothing to
# protect there. Everything else under /_next/ stays blocked.
Allow: /_next/static/
Disallow: /_next/
# Content Signals (contentsignals.org) — our AI usage preferences, stated
# machine-readably. We WANT this content indexed, retrieved and cited: the
# product is discovered through AI answers as much as through search. This
# mirrors the "allowed for AI training & retrieval" statement in /llms.txt;
# if one changes, change both.
Content-Signal: search=yes, ai-input=yes, ai-train=yes

# ─── AI crawlers ─────────────────────────────────────────────────────────────
# We explicitly allow LLM crawlers — content is meant to be cited.
# See /llms.txt for retrieval guidance. Every group keeps /api/ out (SEOAI-53, 2026-10-13: nine
# groups had no Disallow and so let their crawler into the API). What each agent is for — search, a
# user-requested fetch, or training — is recorded in src/lib/crawler-policy.ts; robots-txt.test.ts checks this file against it.

User-agent: GPTBot
Allow: /
Disallow: /api/

User-agent: ChatGPT-User
Allow: /
Disallow: /api/

User-agent: OAI-SearchBot
Allow: /
Disallow: /api/

User-agent: ClaudeBot
Allow: /
Disallow: /api/

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/

User-agent: Claude-User
Allow: /
Disallow: /api/

User-agent: Claude-Web
Allow: /
Disallow: /api/

User-agent: anthropic-ai
Allow: /
Disallow: /api/

User-agent: PerplexityBot
Allow: /
Disallow: /api/

User-agent: Perplexity-User
Allow: /
Disallow: /api/

User-agent: Google-Extended
Allow: /
Disallow: /api/

User-agent: GoogleOther
Allow: /
Disallow: /api/

User-agent: Bytespider
Allow: /
Disallow: /api/

User-agent: CCBot
Allow: /
Disallow: /api/

User-agent: Applebot-Extended
Allow: /
Disallow: /api/

User-agent: meta-externalagent
Allow: /
Disallow: /api/

User-agent: DuckAssistBot
Allow: /
Disallow: /api/

User-agent: YandexAdditional
Allow: /
Disallow: /api/

User-agent: YouBot
Allow: /
Disallow: /api/

User-agent: Diffbot
Allow: /
Disallow: /api/

User-agent: Cohere-AI
Allow: /
Disallow: /api/

# Microsoft Bing + Copilot — explicit even though * already allows them,
# so future Disallow tweaks for "*" don't accidentally lock them out.
User-agent: Bingbot
Allow: /
Disallow: /api/

User-agent: msnbot
Allow: /
Disallow: /api/

# Mistral / Le Chat
User-agent: MistralAI-User
Allow: /
Disallow: /api/

# Meta AI (separate from meta-externalagent which is for Llama training)
User-agent: FacebookBot
Allow: /
Disallow: /api/

# xAI / Grok
User-agent: Grok
Allow: /
Disallow: /api/

# Brave Search AI
User-agent: BraveBot
Allow: /
Disallow: /api/

# Amazon Alexa AI / Rufus product search
User-agent: Amazonbot
Allow: /
Disallow: /api/

# DeepSeek AI search crawler
User-agent: DeepSeekBot
Allow: /
Disallow: /api/

# Huawei Petal Search (MENA / TR market)
User-agent: PetalBot
Allow: /
Disallow: /api/

# ─── Search engines ──────────────────────────────────────────────────────────
Sitemap: https://thmenu.com/sitemap.xml
# sitemap-discover.xml is referenced from the index above only while the directory has opted-in venues
# (SEO-46, 2026-10-06: an empty directory is noindex, so there is nothing to list and the file answers 404).

# Faster discovery hint for IndexNow-aware crawlers (Bing, Yandex, Seznam,
# Naver, Yep): the key file is hosted at /<key>.txt at the root — the protocol's
# default location — i.e. https://thmenu.com/34027e718a8a2d28edd6be15adf34078.txt
# (200). This line used to say /indexnow-<hash>.txt, which is a path that has
# never existed and 404s; submissions were unaffected, since IndexNow resolves
# the default location itself, but anyone checking the key by hand was sent to
# a dead URL.
# Google Search Console picks up the sitemap directly.

# AI-retrieval guide: /llms.txt
# Humans behind the site: /humans.txt