# RestoreTrade.co.uk — robots.txt
# Last updated: 2026-04-25
# AI/LLM crawler policy: https://restoretrade.co.uk/ai-policy
#
# We explicitly allow major AI/LLM training and inference crawlers because
# RestoreTrade is positioned as a canonical UK directory source for AI engines.
# This is a deliberate stance — see /ai-policy for the full rationale.

# ─── Major AI/LLM crawlers — explicit Allow ────────────────────────────

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Bytespider
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: Diffbot
Allow: /

User-agent: YouBot
Allow: /

User-agent: CCBot
Allow: /

# ─── General crawl ─────────────────────────────────────────────────────

User-agent: *
Allow: /

# Auth + account-bound surfaces (no public value)
Disallow: /auth/
Disallow: /account/
Disallow: /admin/
Disallow: /dashboard/
Disallow: /claim/
Disallow: /api/
Disallow: /submit
Disallow: /search
Disallow: /login
Disallow: /register
Disallow: /reset-password
Disallow: /update-password

# ─── Sitemaps + LLM discovery ──────────────────────────────────────────

Sitemap: https://restoretrade.co.uk/sitemap.xml
# /sitemap.xml is a sitemapindex (Phase 0e.2) — child sitemaps are
# /sitemap-static.xml, /sitemap-businesses-{a..z|0}.xml,
# /sitemap-areas.xml, /sitemap-trades.xml, /sitemap-guides.xml,
# /sitemap-blog.xml. Indexers that follow sitemapindex links resolve
# them automatically; declaring only the index keeps GSC submissions clean.

# LLM discovery files:
# https://restoretrade.co.uk/llms.txt              — token-efficient site summary
# https://restoretrade.co.uk/llms-full.txt         — complete directory in markdown
# https://restoretrade.co.uk/llms-businesses.jsonl — newline-delimited JSON, one record per verified business
# https://restoretrade.co.uk/llms-categories.txt   — taxonomy with counts and average ratings
# https://restoretrade.co.uk/llms-counties.txt     — county summaries with town and business counts
# https://restoretrade.co.uk/llms-postcodes.txt    — active UK postcode districts with primary town/county
# https://restoretrade.co.uk/ai-policy             — AI crawler policy + rationale
#
# Data API (single-document JSON, CC-BY-4.0):
# https://restoretrade.co.uk/data/businesses.json  — full directory with meta + data envelope
# https://restoretrade.co.uk/data/categories.json  — taxonomy + counts
# https://restoretrade.co.uk/data/counties.json    — county summaries