diff --git a/.gitignore b/.gitignore index e9651fc..2a63933 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ result # demo/ directory acts as a playground/sandbox, and real git repositories sits there. demo/* !demo/config.yaml +!demo/robots.txt +!demo/robots.txt.license # QEMU disk image. *.qcow2 diff --git a/config.yaml b/config.yaml index 84611c5..1c59a90 100644 --- a/config.yaml +++ b/config.yaml @@ -84,6 +84,10 @@ meta: # NOTE: This applies only to blob page. Code blocks inside README will be untouched. syntaxHighlight: true + # Path to a text file to return for "/robots.txt" requests. + # Omit this option or set empty string to return HTTP 404. + robotsTxt: ./path/to/robots.txt + ui: # Number of commits to display in a single "Commits" (log) page. commitsPageSize: 30 diff --git a/config/config.go b/config/config.go index 8426189..4423fb3 100644 --- a/config/config.go +++ b/config/config.go @@ -35,6 +35,7 @@ type Config struct { Title string `yaml:"title"` Description string `yaml:"description"` SyntaxHighlight bool `yaml:"syntaxHighlight"` + RobotsTxt string `yaml:"robotsTxt"` } `yaml:"meta"` UI struct { CommitsPageSize uint32 `yaml:"commitsPageSize"` @@ -137,6 +138,12 @@ func (c *Config) Resolve(cwd string) error { } } + if c.Meta.RobotsTxt != "" { + if c.Meta.RobotsTxt, err = resolvePath(c.Meta.RobotsTxt, basePath); err != nil { + return err + } + } + if c.UI.CommitsPageSize == 0 { c.UI.CommitsPageSize = 30 } diff --git a/core/core.go b/core/core.go index 8934c36..3387d9e 100644 --- a/core/core.go +++ b/core/core.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "html/template" + "io" "io/fs" "os" "path/filepath" @@ -42,6 +43,8 @@ type Core struct { // t stores compiled templates for caching. tmpl *template.Template templatesDir fs.FS + + robotsTxt string } func New(config *config.Config) (*Core, error) { @@ -81,6 +84,21 @@ func New(config *config.Config) (*Core, error) { return nil, fmt.Errorf("unable to open scan dir: %w", err) } + if config.Meta.RobotsTxt != "" { + file, err := os.Open(config.Meta.RobotsTxt) + if err != nil { + return nil, fmt.Errorf("unable to open robots.txt: %w", err) + } + defer file.Close() + + contents, err := io.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("unable to read robots.txt (%s): %w", config.Meta.RobotsTxt, err) + } + + core.robotsTxt = string(contents) + } + return &core, nil } @@ -120,3 +138,11 @@ func (core *Core) RepositoryPath(name string) (string, error) { func (_ *Core) RepositoryName(path string) string { return strings.TrimSuffix(filepath.Base(path), ".git") } + +func (c *Core) RobotsTxt() io.Reader { + if c.Config.Meta.RobotsTxt == "" { + return nil + } + + return strings.NewReader(c.robotsTxt) +} diff --git a/demo/config.yaml b/demo/config.yaml index 92db552..72f68d0 100644 --- a/demo/config.yaml +++ b/demo/config.yaml @@ -37,6 +37,7 @@ meta: title: legit Demo description: Test legit features with real repositories. syntaxHighlight: true + robotsTxt: ./robots.txt ui: category: diff --git a/demo/robots.txt b/demo/robots.txt new file mode 100644 index 0000000..781c124 --- /dev/null +++ b/demo/robots.txt @@ -0,0 +1,3 @@ +# Demo server for legit, web frontend for git. URLs are not permanent. +User-Agent: * +Disallow: / diff --git a/demo/robots.txt.license b/demo/robots.txt.license new file mode 100644 index 0000000..08ef583 --- /dev/null +++ b/demo/robots.txt.license @@ -0,0 +1,2 @@ +Copyright 2026 Shota FUJI +SPDX-License-Identifier: MIT diff --git a/main.go b/main.go index 272fb19..55eade3 100644 --- a/main.go +++ b/main.go @@ -92,7 +92,7 @@ func main() { log.Fatal(err) } - fsAllowList := make([]filesystemAccess, 2, 10) + fsAllowList := make([]filesystemAccess, 2, 11) fsAllowList[0] = filesystemAccess{ path: c.Repo.ScanPath, isDir: true, @@ -119,6 +119,13 @@ func main() { }) } + if c.Meta.RobotsTxt != "" { + fsAllowList = append(fsAllowList, filesystemAccess{ + path: c.Meta.RobotsTxt, + read: true, + }) + } + if c.Dirs.Static != "" { fsAllowList = append(fsAllowList, filesystemAccess{ path: c.Dirs.Static, diff --git a/pages/pages.go b/pages/pages.go index 6231c80..5fc0c1c 100644 --- a/pages/pages.go +++ b/pages/pages.go @@ -4,6 +4,7 @@ package pages import ( + "io" "net/http" "path" "strings" @@ -36,6 +37,19 @@ func (pages *Pages) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } + if strings.TrimPrefix(r.URL.Path, "/") == "robots.txt" { + reader := pages.core.RobotsTxt() + if reader == nil { + errors.WriteNotFound(pages.core, w, r) + return + } + + w.Header().Add("Content-Type", "text/plain") + w.WriteHeader(http.StatusOK) + io.Copy(w, reader) + return + } + if strings.HasSuffix(r.URL.Path, "/") { r.URL.Path = strings.TrimRight(r.URL.Path, "/") w.Header().Add("Location", r.URL.String())