# Bytespider is an aggressive scraper that ignores crawl etiquette
User-agent: Bytespider
Disallow: /

# Explicitly allowed AI search & answer engines
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: CCBot
Allow: /
Disallow: /settings
Disallow: /*/settings
Disallow: /*/compare
Disallow: /*/fork
Disallow: /search
Disallow: /*/search
Crawl-delay: 1

# Default crawler rules
User-agent: *
Allow: /
Disallow: /settings
Disallow: /*/settings
Disallow: /*/compare
Disallow: /*/fork
Disallow: /search
Disallow: /*/search
Crawl-delay: 1

Sitemap: https://tangled.org/sitemap.xml