# Caissly — the chess library, not the chess arcade.
# https://caissly.com — open editorial, citable.

# Default: allow all.
# /explore/ is NOT blocked here: those pages carry a meta-robots noindex,
# and a blocked URL can still be indexed URL-only (Google can't read the
# noindex if it can't crawl the page) — which is exactly what happened.
# Letting crawlers fetch /explore/ lets them read the noindex and drop it.
# /og/ (social-card images) and /_astro/ (JS/CSS bundles) stay crawlable so
# Google can render pages (mobile-friendly + Core Web Vitals) and fetch
# the social image for rich-result previews.
User-agent: *
Allow: /
# Cloudflare-internal endpoints (email-protection rewrites etc.) 404 for
# crawlers and show up as broken pages in audits — not real content.
Disallow: /cdn-cgi/
# No Crawl-delay. Google ignores the directive; Bing and Yandex honour it,
# and throttling them was pure cost while thousands of URLs sat unvisited.
# The site is static on a CDN — there is no origin to protect.

# ---------------------------------------------------------------
# AI / LLM crawlers
#
# This section and /terms-of-use/ used to disagree: the terms forbid
# scraping the site for bulk training data without written permission,
# while this file waved every training crawler through. Same policy in
# both places now — the split below is the policy, not a preference.
#
# ALLOWED: retrieval. Crawlers that build a search or answer index and
# cite the page with a link when they use it. They send readers back;
# our own analytics has ChatGPT ahead of Google as a referrer. These are
# the reason llms.txt and the direct-answer blocks exist.
#
# BLOCKED: training-only. Crawlers whose sole purpose is ingesting text
# into model weights. No link, no attribution, no reader — and for a
# reference work the ingested fact becomes the substitute for visiting.
# Blocking them costs no traffic because they send none.
#
# GPTBot and ClaudeBot sit across both: OpenAI and Anthropic use them for
# training, but the separation from their retrieval siblings is not clean
# and ChatGPT is currently our best AI referrer. They stay allowed
# deliberately, and /terms-of-use/ names them as the exception.
#
# robots.txt is a request, not an enforcement mechanism. The terms are
# what actually govern; this file states them in a form crawlers read.
# ---------------------------------------------------------------

# --- Retrieval: allowed ---

# OpenAI — ChatGPT-User (live fetch) + OAI-SearchBot (search index).
# GPTBot is training, allowed as the exception noted above.
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic — Claude-Web (live retrieval); ClaudeBot allowed as the
# exception noted above.
User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

# Conventional search. Googlebot also supplies AI Overviews: there is no
# way to appear in Google Search and not in them, so this is not a lever.
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Applebot
Allow: /

User-agent: YandexBot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Baiduspider
Allow: /

User-agent: Amazonbot
Allow: /

# --- Training-only: blocked ---
# Per /terms-of-use/: bulk training use requires written permission.
# Ask at editors@caissly.com and we will consider it.

# Common Crawl — a training corpus with no retrieval product.
User-agent: CCBot
Disallow: /

# Meta AI training.
User-agent: Meta-ExternalAgent
Disallow: /

# Gemini training opt-out. Does not affect Googlebot, Search ranking, or
# AI Overviews, which run off the ordinary index.
User-agent: Google-Extended
Disallow: /

# Apple Intelligence training opt-out. Applebot itself stays allowed, so
# Siri and Spotlight results are unaffected.
User-agent: Applebot-Extended
Disallow: /

# Deprecated Anthropic token, kept blocked so a legacy client reading it
# does not treat the absence as permission.
User-agent: anthropic-ai
Disallow: /

# ---------------------------------------------------------------
# Sitemaps
# ---------------------------------------------------------------

Sitemap: https://caissly.com/sitemap-index.xml