# spine.md robots.txt — optimized 2025-11-05
# Goal: fast, clean SEO crawling + explicit AI/LLM policy

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# SEO hygiene: avoid thin/duplicative pages
User-agent: *
Disallow: /?s=
Disallow: /search/
Disallow: /*?replytocom=*
Disallow: /*?utm_=*

# --- AI / LLM Crawlers: Explicit ALLOW ---
User-agent: GPTBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: CCBot
Allow: /

# --- Sitemaps ---
https://www.spine.md/sitemap_index.xml