# Readfeed Robots.txt
# https://www.readfeed.com

User-agent: *
Allow: /

# Disallow internal/test pages
# (/_next/ is deliberately NOT disallowed: Google must fetch JS/CSS to render pages;
#  blocking it makes every static page look broken to the renderer.)
Disallow: /api/
Disallow: /error-test
Disallow: /invite-test
Disallow: /test-notifications
Disallow: /tokenlogin
Disallow: /loading
Disallow: /profile/edit

# Disallow low-value tag and category pages (thin content)
Disallow: /blog/category/
Disallow: /blog/tag/

# Disallow auth pages (no SEO value)
Disallow: /login
Disallow: /registration
Disallow: /forgot-password
Disallow: /my-bookclubs
Disallow: /publishers-only
Disallow: /subscribe
Disallow: /data-deletion

# AI training crawlers: allowed but rate-limited (high origin load, no referral traffic).
# A named group means these bots ignore the "*" rules above, so the disallows are repeated here.
User-agent: ClaudeBot
Crawl-delay: 30
Disallow: /api/
Disallow: /error-test
Disallow: /invite-test
Disallow: /test-notifications
Disallow: /tokenlogin
Disallow: /loading
Disallow: /_next/
Disallow: /profile/edit
Disallow: /blog/category/
Disallow: /blog/tag/
Disallow: /login
Disallow: /registration
Disallow: /forgot-password
Disallow: /my-bookclubs
Disallow: /publishers-only
Disallow: /subscribe
Disallow: /data-deletion

# Sitemap location
Sitemap: https://www.readfeed.com/sitemap.xml