# Train Walk Poop - robots.txt
# Updated to allow AI search crawlers while blocking bulk scrapers and AI training bots.

# Content signals: allow search indexing and AI search answers, block AI training
User-agent: *
Content-Signal: search=yes,ai-input=yes,ai-train=no
Allow: /
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# -- AI SEARCH CRAWLERS (allowed) --
# These crawlers power AI-generated search answers in ChatGPT, Claude,
# Google AI Overviews, and Perplexity. Allowing them helps your business
# appear in AI search results.

User-agent: GPTBot
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

# -- OPENAI ADS & SEARCH CRAWLERS (allowed, required for ChatGPT Ads) --
User-agent: OAI-AdsBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

# -- BULK SCRAPERS & AI TRAINING BOTS (blocked) --
# These bots collect content for mass AI model training, not search answers.

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: CloudflareBrowserRenderingCrawler
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Sitemap
Sitemap: https://trainwalkpoop.com/sitemap_index.xml