diff --git a/web/static/robots.txt b/web/static/robots.txt
index b6dd6670c..ec9c7b918 100644
--- a/web/static/robots.txt
+++ b/web/static/robots.txt
@@ -1,3 +1,32 @@
-# allow crawling everything by default
+# Bytespider is an aggressive scraper that ignores crawl etiquette
+User-agent: Bytespider
+Disallow: /
+
+# Explicitly allowed AI search & answer engines
+User-agent: GPTBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: PerplexityBot
+User-agent: Google-Extended
+User-agent: CCBot
+Allow: /
+Disallow: /settings
+Disallow: /*/settings
+Disallow: /*/compare
+Disallow: /*/fork
+Disallow: /search
+Disallow: /*/search
+Crawl-delay: 1
+
+# Default crawler rules
 User-agent: *
-Disallow:
+Allow: /
+Disallow: /settings
+Disallow: /*/settings
+Disallow: /*/compare
+Disallow: /*/fork
+Disallow: /search
+Disallow: /*/search
+Crawl-delay: 1
+
+Sitemap: https://tangled.org/sitemap.xml