From 7af3d3d87cec07fb263de5e82f080112bff886d5 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Mon, 09 Feb 2026 13:39:54 +0000 Subject: [PATCH] appview/state: update robots.txt Signed-off-by: Anirudh Oppiliappan --- appview/state/state.go | 9 ++++++++- 1 file(s) changed, 8 insertion(s)(+), 1 deletion(s)(-) diff --git a/appview/state/state.go b/appview/state/state.go --- a/appview/state/state.go +++ b/appview/state/state.go @@ -207,8 +207,15 @@ func (s *State) RobotsTxt(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/plain") w.Header().Set("Cache-Control", "public, max-age=86400") // one day - robotsTxt := `User-agent: * + robotsTxt := `# Hello, Tanglers! +User-agent: * Allow: / +Disallow: /*/*/settings +Disallow: /settings +Disallow: /*/*/compare +Disallow: /*/*/fork + +Crawl-delay: 1 ` w.Write([]byte(robotsTxt)) } -- tangled.sh