# GunSpec.io - Robots.txt
#
# A crawler obeys exactly ONE group: the most specific User-agent match. Groups
# are NOT inherited. Googlebot previously had its own group containing only
# "Allow: /", so none of the Disallow rules further down applied to it -- it was
# free to crawl /v1/ (including the invisible footer honeypot links, reachable
# from 6,000+ pages) and the private /profile and /login paths.
#
# So the crawlers we allow are listed together in ONE group that also carries
# the shared Disallow set, rather than in bare "Allow: /" groups that silently
# opt them out of every restriction.

# --- Search engines and AI answer engines that send readers back ---
# The same crawlers the site renders pages for (apps/site/src/worker/audience.ts,
# KEEP_GROUPS), checked against this file by a test. Google-Extended and
# Applebot-Extended are opt-out tokens, not crawlers: Googlebot and Applebot do
# the fetching, so they cost nothing here either way.
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Google-InspectionTool
User-agent: Google-Extended
User-agent: Bingbot
User-agent: BingPreview
User-agent: Applebot
User-agent: Applebot-Extended
User-agent: Yandex
User-agent: DuckDuckBot
User-agent: SeznamBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: meta-webindexer
User-agent: YouBot
Allow: /
# The API is not a crawl surface. The footer also carries invisible honeypot
# links under /v1/ meant to catch scrapers, not search engines.
Disallow: /v1/
# Record photographs are pages' images, not API data: Google Images needs them.
Allow: /v1/firearms/*/images/
Disallow: /openapi.json
# Private / authenticated areas.
Disallow: /login
Disallow: /profile
Disallow: /admin
Disallow: /onboarding
Disallow: /auth/
Disallow: /billing/
Disallow: /webhooks/
Disallow: /user/
Disallow: /*/login
Disallow: /*/profile
Disallow: /*/admin
Disallow: /*/onboarding
# Share links are unlisted by design.
Disallow: /*/shared/

# --- Pushed to the docs: AI training, dataset collection, and search for markets we do not serve ---
# These send no readers back, and a page they read costs a server render. The
# catalogue is closed to them; /llms.txt is open, and so is the whole of the
# docs host (its own robots.txt allows everything), which says what the data is
# and how to get it properly. Allow comes first and is the longer match, so a
# crawler that reads either order still reaches /llms.txt.
# The worker enforces the same list for the ones that do not read this file.
User-agent: GPTBot
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
User-agent: Amazonbot
User-agent: meta-externalagent
User-agent: Reflectionbot
User-agent: CCBot
User-agent: cohere-ai
User-agent: cohere-training-data-crawler
User-agent: Diffbot
User-agent: ImagesiftBot
User-agent: Omgilibot
User-agent: Omgili
User-agent: Baiduspider
User-agent: Sogou
Allow: /llms.txt
Disallow: /

# --- Block known scraping / SEO bots ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: SERankingBacklinksBot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SEOkicks
Disallow: /

User-agent: Seekport
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: Rogerbot
Disallow: /

User-agent: Zoominfobot
Disallow: /

User-agent: MojeekBot
Disallow: /

# --- Everything else: same restrictions, plus a crawl delay ---
User-agent: *
Allow: /
Disallow: /v1/
# Record photographs are pages' images, not API data: Google Images needs them.
Allow: /v1/firearms/*/images/
Disallow: /openapi.json
Disallow: /login
Disallow: /profile
Disallow: /admin
Disallow: /onboarding
Disallow: /auth/
Disallow: /billing/
Disallow: /webhooks/
Disallow: /user/
Disallow: /*/login
Disallow: /*/profile
Disallow: /*/admin
Disallow: /*/onboarding
Disallow: /*/shared/
Crawl-delay: 10

# Sitemaps
# The blog and docs are separate apps on their own subdomains; each hosts its
# own sitemap. Listed here too so a crawler that only reads the apex robots.txt
# still discovers them (both hosts are verified in Search Console).
Sitemap: https://gunspec.io/sitemap.xml
Sitemap: https://blog.gunspec.io/sitemap.xml
Sitemap: https://docs.gunspec.io/sitemap.xml

# LLMs.txt
# https://gunspec.io/llms.txt