# Broadway Podcast Network - robots.txt

# Allow all crawlers by default with rate limiting
User-Agent: *
Allow: /
Crawl-delay: 1

# Disallow internal/admin paths
Disallow: /*/*/*/customFields
Disallow: /*/*/meta
Disallow: /unknown/*
Disallow: /api/
Disallow: /admin/

# Block aggressive SEO/marketing bots entirely (they provide no value)
User-Agent: AhrefsBot
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: MJ12bot
Disallow: /

User-Agent: DotBot
Disallow: /

User-Agent: BLEXBot
Disallow: /

User-Agent: PetalBot
Disallow: /

# Block TikTok crawler - extremely aggressive
User-Agent: Bytespider
Disallow: /

# Block other known aggressive crawlers
User-Agent: MegaIndex
Disallow: /

User-Agent: DataForSeoBot
Disallow: /

User-Agent: BomboraBot
Disallow: /

User-Agent: Amazonbot
Crawl-delay: 10

# Major search engines - allow with reasonable rate limiting
User-Agent: Googlebot
Allow: /
Crawl-delay: 1

User-Agent: Bingbot
Allow: /
Crawl-delay: 2

User-Agent: Slurp
Allow: /
Crawl-delay: 5

User-Agent: DuckDuckBot
Allow: /
Crawl-delay: 2

# LLM crawlers - allow with rate limiting
User-Agent: GPTBot
Allow: /
Crawl-delay: 5

User-Agent: ChatGPT-User
Allow: /
Crawl-delay: 5

User-Agent: Claude-Web
Allow: /
Crawl-delay: 5

User-Agent: Anthropic-AI
Allow: /
Crawl-delay: 5

User-Agent: ClaudeBot
Allow: /
Crawl-delay: 5

User-Agent: PerplexityBot
Allow: /
Crawl-delay: 5

User-Agent: Applebot-Extended
Allow: /
Crawl-delay: 5

User-Agent: Google-Extended
Allow: /
Crawl-delay: 5

# Sitemap location
Sitemap: https://broadwaypodcastnetwork.com/sitemap.xml