From 7b05aecbe2a19a2f013bd1b4f7e9284d880ca137 Mon Sep 17 00:00:00 2001 From: dawn Date: Sat, 5 Sep 2026 02:59:01 +0900 Subject: [PATCH] web: update robots.txt for search engines Signed-off-by: dawn --- web/static/robots.txt | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/web/static/robots.txt b/web/static/robots.txt index b6dd6670c..ec9c7b918 100644 --- a/web/static/robots.txt +++ b/web/static/robots.txt @@ -1,3 +1,32 @@ -# allow crawling everything by default +# Bytespider is an aggressive scraper that ignores crawl etiquette +User-agent: Bytespider +Disallow: / + +# Explicitly allowed AI search & answer engines +User-agent: GPTBot +User-agent: ChatGPT-User +User-agent: ClaudeBot +User-agent: PerplexityBot +User-agent: Google-Extended +User-agent: CCBot +Allow: / +Disallow: /settings +Disallow: /*/settings +Disallow: /*/compare +Disallow: /*/fork +Disallow: /search +Disallow: /*/search +Crawl-delay: 1 + +# Default crawler rules User-agent: * -Disallow: +Allow: / +Disallow: /settings +Disallow: /*/settings +Disallow: /*/compare +Disallow: /*/fork +Disallow: /search +Disallow: /*/search +Crawl-delay: 1 + +Sitemap: https://tangled.org/sitemap.xml -- 2.51.2