diff --git a/appview/codesearch/codesearch.go b/appview/codesearch/codesearch.go new file mode 100644 index 00000000..18a39307 --- /dev/null +++ b/appview/codesearch/codesearch.go @@ -0,0 +1,267 @@ +package codesearch + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/sourcegraph/zoekt" + "github.com/sourcegraph/zoekt/query" + "tangled.org/core/appview/models" + "tangled.org/core/appview/pagination" +) + +type CodeSearch struct { + Host string // zoekt-webserver host. example: https://zoekt.example.com + Client *http.Client +} + +func (s *CodeSearch) GetClient() *http.Client { + if s.Client != nil { + return s.Client + } + return http.DefaultClient +} + +type RepoOnlyError struct{ Query string } + +func (e *RepoOnlyError) Error() string { + return "query only filters by repo name; use repo search instead" +} + +// jsonSearchArgs mirrors zoekt's /api/search request body. +type jsonSearchArgs struct { + Q string + Opts *zoekt.SearchOptions +} + +// jsonSearchReply mirrors zoekt's /api/search response body. +type jsonSearchReply struct { + Result *zoekt.SearchResult +} + +// jsonListArgs mirrors zoekt's /api/list request body. +type jsonListArgs struct { + Q string + Opts *zoekt.ListOptions +} + +// jsonListReply mirrors zoekt's /api/list response body. +type jsonListReply struct { + List *zoekt.RepoList +} + +// SearchResults is a single page of content-search results plus whether more +// pages follow. +type SearchResults struct { + Results []models.Result + HasMore bool + Stats zoekt.Stats // zoekt search stats (MatchCount, FileCount, Duration, …) +} + +// Search queries zoekt server for FileNameMatch or ChunkMatch. +// It returns *RepoOnlyError when the query only filters by repo name +// (optionally with `lang:` filter.) +func (s *CodeSearch) Search(ctx context.Context, queryStr string, page pagination.Page) (*SearchResults, error) { + q, err := query.Parse(queryStr) + if err != nil { + return nil, fmt.Errorf("parse query: %w", err) + } + if rs, ok := asRepoSearch(q); ok { + return nil, &RepoOnlyError{Query: rs.Query()} + } + + opts := &zoekt.SearchOptions{ + ChunkMatches: true, + MaxWallTime: 10 * time.Second, + NumContextLines: 2, + } + if page.Limit > 0 { + // +1 so we can detect a following page. + opts.MaxDocDisplayCount = page.Offset + page.Limit + 1 + } + + body, err := json.Marshal(jsonSearchArgs{ + Q: queryStr, + Opts: opts, + }) + if err != nil { + return nil, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(s.Host, "/") + "/api/search" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.GetClient().Do(req) + if err != nil { + return nil, fmt.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return nil, fmt.Errorf("zoekt search: status %d: %s", resp.StatusCode, strings.TrimSpace(string(b))) + } + + var reply jsonSearchReply + if err := json.NewDecoder(resp.Body).Decode(&reply); err != nil { + return nil, fmt.Errorf("decode response: %w", err) + } + if reply.Result == nil { + return &SearchResults{}, nil + } + + stats := reply.Result.Stats + all := toResults(reply.Result) + end := page.Offset + page.Limit + if page.Limit <= 0 { + // No window requested: return everything. + return &SearchResults{Results: all, Stats: stats}, nil + } + + hasMore := len(all) > end // extra card present ⇒ more pages + if page.Offset >= len(all) { + return &SearchResults{HasMore: false, Stats: stats}, nil + } + if end > len(all) { + end = len(all) + } + return &SearchResults{Results: all[page.Offset:end], HasMore: hasMore, Stats: stats}, nil +} + +// RepoCount returns the total number of repositories in the zoekt index. +func (s *CodeSearch) RepoCount(ctx context.Context) (int, error) { + body, err := json.Marshal(jsonListArgs{ + Q: "", // empty query ⇒ match all repos + Opts: &zoekt.ListOptions{Field: zoekt.RepoListFieldRepos}, + }) + if err != nil { + return 0, fmt.Errorf("marshal request: %w", err) + } + + url := strings.TrimRight(s.Host, "/") + "/api/list" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + if err != nil { + return 0, fmt.Errorf("build request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + + resp, err := s.GetClient().Do(req) + if err != nil { + return 0, fmt.Errorf("do request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return 0, fmt.Errorf("zoekt list: status %d: %s", resp.StatusCode, strings.TrimSpace(string(b))) + } + + var reply jsonListReply + if err := json.NewDecoder(resp.Body).Decode(&reply); err != nil { + return 0, fmt.Errorf("decode response: %w", err) + } + if reply.List == nil { + return 0, nil + } + return reply.List.Stats.Repos, nil +} + +// toResults maps zoekt FileMatches into local Results +func toResults(sr *zoekt.SearchResult) []models.Result { + var out []models.Result + for _, fm := range sr.Files { + // HACK: zoekt use int64 repo.ID as identifier, but we expect DID (string) as an repo identifier. + // as a quick hack without patching zoekt, we extract the DID from RepoURLs + repoDID := extractDID(sr.RepoURLs[fm.Repository]) + res := models.Result{ + RepoDID: repoDID, + FilePath: fm.FileName, + Branches: fm.Branches, + Commit: fm.Version, + Language: fm.Language, + } + for _, cm := range fm.ChunkMatches { + if cm.FileName { + res.File = &models.Result_FileMatch{Ranges: cm.Ranges} + break + } else { + res.Chunks = append(res.Chunks, models.Result_ChunkMatch{ + Content: string(cm.Content), + ContentStartLine: int(cm.ContentStart.LineNumber), + Ranges: cm.Ranges, + }) + } + } + out = append(out, res) + } + return out +} + +// extractDID pulls the repo DID out of a zoekt FileURLTemplate of the form +// "{appviewURL}/{repoDID}/blob/{commit}/{path}". +func extractDID(urlTemplate string) syntax.DID { + if urlTemplate == "" { + return "" + } + u, err := url.Parse(urlTemplate) + if err != nil { + return "" + } + seg := strings.SplitN(strings.TrimPrefix(u.Path, "/"), "/", 2)[0] + return syntax.DID(seg) +} + +type repoSearchQuery struct { + RepoNames []string + Language string +} + +func (r repoSearchQuery) Query() string { + parts := append([]string{}, r.RepoNames...) + if r.Language != "" { + parts = append(parts, "lang:"+r.Language) + } + return strings.Join(parts, " ") +} + +func asRepoSearch(q query.Q) (repoSearchQuery, bool) { + var rs repoSearchQuery + if t, ok := q.(*query.Type); ok && t.Type == query.TypeRepo { + query.VisitAtoms(t.Child, func(a query.Q) { + switch v := a.(type) { + case *query.Repo: + rs.RepoNames = append(rs.RepoNames, v.Regexp.String()) + case *query.Substring: + rs.RepoNames = append(rs.RepoNames, v.Pattern) + case *query.Language: + rs.Language = v.Language + } + }) + return rs, true + } + hasRepo, only := false, true + query.VisitAtoms(q, func(a query.Q) { + switch v := a.(type) { + case *query.Repo: + hasRepo = true + rs.RepoNames = append(rs.RepoNames, v.Regexp.String()) + case *query.Language: + rs.Language = v.Language + default: + only = false + } + }) + return rs, hasRepo && only +} diff --git a/appview/codesearch/codesearch_test.go b/appview/codesearch/codesearch_test.go new file mode 100644 index 00000000..ae0b8376 --- /dev/null +++ b/appview/codesearch/codesearch_test.go @@ -0,0 +1,61 @@ +package codesearch + +import ( + "testing" + + "github.com/sourcegraph/zoekt/query" +) + +func TestAsRepoSearch(t *testing.T) { + cases := []struct { + query string + wantOK bool + wantStr string // rewritten repo-search query, only checked when wantOK + }{ + {"repo:foo", true, "foo"}, + {"repo:foo repo:bar", true, "foo bar"}, + {"repo:foo lang:go", true, "foo lang:Go"}, + {"lang:go", false, ""}, + {"repo:foo bar", false, ""}, + {"repo:foo file:x", false, ""}, + {"file:x", false, ""}, + {"type:repo foo", true, "foo"}, + {"foo", false, ""}, + {"branch:main", false, ""}, + } + + for _, tc := range cases { + t.Run(tc.query, func(t *testing.T) { + q, err := query.Parse(tc.query) + if err != nil { + t.Fatalf("parse %q: %v", tc.query, err) + } + rs, ok := asRepoSearch(q) + if ok != tc.wantOK { + t.Fatalf("asRepoSearch(%q) ok = %v, want %v", tc.query, ok, tc.wantOK) + } + if ok && rs.Query() != tc.wantStr { + t.Errorf("asRepoSearch(%q).Query() = %q, want %q", tc.query, rs.Query(), tc.wantStr) + } + }) + } +} + +func TestExtractDID(t *testing.T) { + cases := []struct { + tmpl string + want string + }{ + {"https://tangled.org/did:plc:abc123/blob/{{.Version}}/{{.Path}}", "did:plc:abc123"}, + {"http://localhost:3000/did:web:example.com/blob/{{.Version}}/{{.Path}}", "did:web:example.com"}, + {"", ""}, + } + + for _, tc := range cases { + t.Run(tc.tmpl, func(t *testing.T) { + if got := extractDID(tc.tmpl); string(got) != tc.want { + t.Errorf("extractDID(%q) = %q, want %q", tc.tmpl, got, tc.want) + } + }) + } +} diff --git a/appview/config/config.go b/appview/config/config.go index 36a66a68..a3c4f650 100644 --- a/appview/config/config.go +++ b/appview/config/config.go @@ -157,6 +157,10 @@ type OgreConfig struct { Host string `env:"HOST, default=https://ogre.tangled.network"` } +type CodeSearchConfig struct { + ZoektUrl string `env:"ZOEKT_URL"` +} + type SSHConfig struct { Enabled bool `env:"ENABLED, default=false"` ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:3333"` @@ -198,6 +202,7 @@ type Config struct { KnotMirror KnotMirrorConfig `env:",prefix=TANGLED_KNOTMIRROR_"` Ogre OgreConfig `env:",prefix=TANGLED_OGRE_"` SSH SSHConfig `env:",prefix=TANGLED_SSH_"` + CodeSearch CodeSearchConfig `env:",prefix=TANGLED_CODESEARCH_"` } func LoadConfig(ctx context.Context) (*Config, error) { diff --git a/appview/models/codesearch.go b/appview/models/codesearch.go new file mode 100644 index 00000000..ed6e8098 --- /dev/null +++ b/appview/models/codesearch.go @@ -0,0 +1,46 @@ +package models + +import ( + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/sourcegraph/zoekt" +) + +// Result is a single matched file. A zoekt FileMatch is either a filename +// match or a set of content matches, never both: File is set for the former, +// Chunks for the latter. +type Result struct { + RepoDID syntax.DID + FilePath string + Branches []string // branch names + Commit string // commit id + Language string + + File *Result_FileMatch // set for a filename match + Chunks []Result_ChunkMatch // set for content matches +} + +type Result_FileMatch struct { + // Ranges are the matched span(s) within FilePath. LineNumber is always 1 for + // filename matches; Column is 1-based and in runes. + Ranges []zoekt.Range +} + +type Result_ChunkMatch struct { + // Content is a contiguous run of complete lines that fully contains Ranges. + Content string + // ContentStartLine is the 1-based line number of Content's first line. + ContentStartLine int + // Ranges are the matched span(s) within the file. LineNumber/Column are + // 1-based, Column is in runes. A Range may span multiple lines. + Ranges []zoekt.Range +} + +// IsFileMatch tells if search result is from file-name match +func (r *Result) IsFileMatch() bool { + return r.File != nil +} + +// IsChunkMatch tells if search result is from chunk match +func (r *Result) IsChunkMatch() bool { + return len(r.Chunks) > 0 +} diff --git a/appview/pages/codesearch.go b/appview/pages/codesearch.go new file mode 100644 index 00000000..bf2f933d --- /dev/null +++ b/appview/pages/codesearch.go @@ -0,0 +1,49 @@ +package pages + +import "sort" + +// helper functions to render search match highlights + +// mergeIntervals sorts half-open rune intervals and merges overlapping/adjacent ones. +func mergeIntervals(in [][2]int) [][2]int { + if len(in) < 2 { + return in + } + sort.Slice(in, func(i, j int) bool { return in[i][0] < in[j][0] }) + out := in[:1] + for _, iv := range in[1:] { + last := &out[len(out)-1] + if iv[0] <= last[1] { + if iv[1] > last[1] { + last[1] = iv[1] + } + continue + } + out = append(out, iv) + } + return out +} + +// spanRunes splits runes into alternating unmatched/matched ChunkSpans using the +// (sorted, merged) match intervals. Returns nil for an empty line. +func spanRunes(runes []rune, intervals [][2]int) []ChunkSpan { + if len(runes) == 0 { + return nil + } + if len(intervals) == 0 { + return []ChunkSpan{{Text: string(runes)}} + } + var spans []ChunkSpan + pos := 0 + for _, iv := range intervals { + if iv[0] > pos { + spans = append(spans, ChunkSpan{Text: string(runes[pos:iv[0]])}) + } + spans = append(spans, ChunkSpan{Text: string(runes[iv[0]:iv[1]]), Match: true}) + pos = iv[1] + } + if pos < len(runes) { + spans = append(spans, ChunkSpan{Text: string(runes[pos:])}) + } + return spans +} diff --git a/appview/pages/pages.go b/appview/pages/pages.go index 5c0353ab..62fbef10 100644 --- a/appview/pages/pages.go +++ b/appview/pages/pages.go @@ -33,6 +33,7 @@ import ( "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/go-git/go-git/v5/plumbing" + "github.com/sourcegraph/zoekt" ) //go:embed templates/* static legal @@ -1682,16 +1683,19 @@ func (p *Pages) SingleString(w io.Writer, params SingleStringParams) error { type SearchReposParams struct { BaseParams - Repos []models.Repo + FilterType string // "repo" | "code" + Repos []SearchResult Page pagination.Page ResultCount int FilterQuery string SortParam string TimeTaken time.Duration DocCount int64 + ErrorMsg string } func (p *Pages) SearchRepos(w io.Writer, params SearchReposParams) error { + params.FilterType = "repo" return p.execute("search/search", w, params) } @@ -1713,6 +1717,131 @@ func (p *Pages) SearchQuickMobile(w io.Writer, params SearchQuickParams) error { return tpl.ExecuteTemplate(w, "search/fragments/quickMobile", params) } +type SearchResult struct { + RepoDID syntax.DID + Repo *models.Repo + FilePath string + Branches []string + Commit string + Language string + + File *CodeSearchResult_File // filename match + Chunks CodeSearchResult_Chunks // content matches +} + +// CodeSearchResult_Chunk is a content match with its lines pre-rendered. +type CodeSearchResult_Chunk struct { + Lines []ChunkLine // precomputed from Content/ContentStartLine/Ranges + MatchCount int // number of match ranges in this chunk +} + +type CodeSearchResult_Chunks []CodeSearchResult_Chunk + +func (cs CodeSearchResult_Chunks) MatchCount() int { + count := 0 + for _, c := range cs { + count += c.MatchCount + } + return count +} + +type CodeSearchResult_File struct { + NameSpans []ChunkSpan // precomputed from FilePath/Ranges +} + +type ChunkSpan struct { + Text string + Match bool +} + +type ChunkLine struct { + Num int + Spans []ChunkSpan + Highlight bool +} + +// ChunkLines renders a chunk's Content into per-line ChunkLines, splitting each +// line into matched/unmatched spans using ranges. startLine is the 1-based line +// number of the first line. +func ChunkLines(content string, startLine int, ranges []zoekt.Range) []ChunkLine { + if startLine < 1 { + startLine = 1 + } + // trim a single trailing newline so we don't emit a spurious empty line + content = strings.TrimSuffix(content, "\n") + lines := strings.Split(content, "\n") + out := make([]ChunkLine, len(lines)) + for i, text := range lines { + num := startLine + i + runes := []rune(text) + + // collect matched rune intervals [c0,c1) for this line + var intervals [][2]int + for _, rg := range ranges { + if num < int(rg.Start.LineNumber) || num > int(rg.End.LineNumber) { + continue + } + c0, c1 := 0, len(runes) + if num == int(rg.Start.LineNumber) { + c0 = int(rg.Start.Column) - 1 + } + if num == int(rg.End.LineNumber) { + c1 = int(rg.End.Column) - 1 + } + c0 = max(0, min(c0, len(runes))) + c1 = max(0, min(c1, len(runes))) + if c0 < c1 { + intervals = append(intervals, [2]int{c0, c1}) + } + } + intervals = mergeIntervals(intervals) + + out[i] = ChunkLine{ + Num: num, + Spans: spanRunes(runes, intervals), + Highlight: len(intervals) > 0, + } + } + return out +} + +// FileNameSpans splits a filename into matched/unmatched spans using ranges. +// Filename ranges live on line 1; columns are clamped to rune bounds. +func FileNameSpans(name string, ranges []zoekt.Range) []ChunkSpan { + runes := []rune(name) + var intervals [][2]int + for _, rg := range ranges { + if rg.Start.LineNumber > 1 || rg.End.LineNumber < 1 { + continue + } + c0 := max(0, min(int(rg.Start.Column)-1, len(runes))) + c1 := max(0, min(int(rg.End.Column)-1, len(runes))) + if c0 < c1 { + intervals = append(intervals, [2]int{c0, c1}) + } + } + return spanRunes(runes, mergeIntervals(intervals)) +} + +type CodeSearchParams struct { + BaseParams + FilterType string // "code" + FilterQuery string + Results []SearchResult + Page pagination.Page + HasMore bool + ErrorMsg string + + MatchCount int + FileCount int + TimeTaken time.Duration +} + +func (p *Pages) CodeSearch(w io.Writer, params CodeSearchParams) error { + params.FilterType = "code" + return p.execute("search/search", w, params) +} + func (p *Pages) Home(w io.Writer, params TimelineParams) error { return p.execute("timeline/home", w, params) } diff --git a/appview/pages/templates/fragments/pagination.html b/appview/pages/templates/fragments/pagination.html index 65ed2365..b89686d4 100644 --- a/appview/pages/templates/fragments/pagination.html +++ b/appview/pages/templates/fragments/pagination.html @@ -1,13 +1,22 @@ {{ define "fragments/pagination" }} - {{/* Params: Page (pagination.Page), TotalCount (int), BasePath (string), QueryParams (url.Values) */}} + {{/* Params: Page (pagination.Page), BasePath (string), QueryParams (url.Values), TotalCount (int)|HasMore (bool) */}} + {{/* Cursor mode when HasMore is provided */}} + {{ $page := .Page }} {{ $totalCount := .TotalCount }} {{ $basePath := .BasePath }} {{ $queryParams := safeUrl .QueryParams.Encode }} + {{ $cursor := mapContains . "HasMore" }} {{ $prev := $page.Previous.Offset }} {{ $next := $page.Next.Offset }} - {{ $lastPage := sub $totalCount (mod $totalCount $page.Limit) }} + + {{ $hasNext := false }} + {{ if $cursor }} + {{ $hasNext = .HasMore }} + {{ else }} + {{ $hasNext = lt $next $totalCount }} + {{ end }}
- {{ if gt $page.Offset 0 }} - - 1 - - {{ end }} + {{ if not $cursor }} + {{ $lastPage := sub $totalCount (mod $totalCount $page.Limit) }} - {{ if gt $prev $page.Limit }} - — - {{ end }} + {{ if gt $page.Offset 0 }} + + 1 + + {{ end }} - {{ if gt $prev 0 }} - - {{ add (div $prev $page.Limit) 1 }} - - {{ end }} + {{ if gt $prev $page.Limit }} + — + {{ end }} - - {{ add (div $page.Offset $page.Limit) 1 }} - + {{ if gt $prev 0 }} + + {{ add (div $prev $page.Limit) 1 }} + + {{ end }} - {{ if lt $next $lastPage }} - - {{ add (div $next $page.Limit) 1 }} - - {{ end }} + + {{ add (div $page.Offset $page.Limit) 1 }} + - {{ if lt $next (sub $totalCount (mul 2 $page.Limit)) }} - — - {{ end }} + {{ if lt $next $lastPage }} + + {{ add (div $next $page.Limit) 1 }} + + {{ end }} - {{ if lt $page.Offset $lastPage }} - - {{ add (div $lastPage $page.Limit) 1 }} - + {{ if lt $next (sub $totalCount (mul 2 $page.Limit)) }} + — + {{ end }} + + {{ if lt $page.Offset $lastPage }} + + {{ add (div $lastPage $page.Limit) 1 }} + + {{ end }} {{ end }} + {{ range $chunk.Lines }} +
+ {{ .Num }} +
+ {{- range .Spans -}} + {{- if .Match -}} + {{ .Text }} + {{- else -}} + {{ .Text }} + {{- end -}} + {{- end -}} + {{- if not .Spans }}​{{ end -}} +
+
+ {{ end }} +
+{{ end }} diff --git a/appview/pages/templates/search/fragments/resultCard.html b/appview/pages/templates/search/fragments/resultCard.html new file mode 100644 index 00000000..df2caf3b --- /dev/null +++ b/appview/pages/templates/search/fragments/resultCard.html @@ -0,0 +1,110 @@ +{{ define "search/fragments/resultCard" }} + {{ $owner := resolve .Repo.Did }} + {{ $slug := .Repo.Slug }} + +
+
+
+ {{ template "user/fragments/pic" (list .Repo.Did "size-5") }} + {{ $owner }}/{{ $slug }} +
+
+ {{ range .Branches }} + {{ . }} + {{ end }} +
+ {{ with .Repo.RepoStats }} +
+ {{ i "star" "size-4" }} + {{ scaleFmt .StarCount }} +
+ {{ end }} +
+ + {{ if .File }} + {{/* filename match */}} +
+
+ {{ i "file" "size-4 flex-shrink-0" }} + + {{- range .File.NameSpans -}} + {{- if .Match -}}{{ .Text }} + {{- else -}}{{ .Text }}{{- end -}} + {{- end -}} + +
+
+ {{ with .Language }} + {{ template "repo/fragments/colorBall" (dict "color" (langColor .)) }} + {{ . }} + {{ end }} +
+
+ {{ else if .Chunks }} + {{/* chunk match */}} +
+
+
+ {{ i "file" "size-4 flex-shrink-0" }} + {{ .FilePath }} +
+
+ {{ with .Language }} + {{ template "repo/fragments/colorBall" (dict "color" (langColor .)) }} + {{ . }} + {{ end }} +
+
+ {{ range $i, $chunk := .Chunks }} + {{ if lt $i 3 }} + {{ if gt $i 0 }} +
+ ··· +
+ {{ end }} + {{ template "search/fragments/chunkBody" (dict "Owner" $owner "Slug" $slug "Commit" $.Commit "FilePath" $.FilePath "Chunk" $chunk) }} + {{ end }} + {{ end }} + {{ if gt (len .Chunks) 3 }} +
+ +
+ {{ i "chevron-down" "size-3 m-1" }} + Show {{ (slice .Chunks 3).MatchCount }} more matches +
+ + + {{ range $i, $chunk := .Chunks }} + {{ if ge $i 3 }} +
+ ··· +
+ {{ template "search/fragments/chunkBody" (dict "Owner" $owner "Slug" $slug "Commit" $.Commit "FilePath" $.FilePath "Chunk" $chunk) }} + {{ end }} + {{ end }} +
+ {{ end }} +
+ {{ else }} + {{/* repo match */}} +
+ {{ with .Repo.Description }} +
+ {{ . | description }} +
+ {{ end }} + {{ with .Repo.RepoStats }} + {{ with .Language }} +
+ {{ template "repo/fragments/colorBall" (dict "color" (langColor .)) }} + {{ . }} +
+ {{ end }} + {{ end }} +
+ {{ end }} +
+{{ end }} diff --git a/appview/pages/templates/search/search.html b/appview/pages/templates/search/search.html index 2ce015dc..18b677f2 100644 --- a/appview/pages/templates/search/search.html +++ b/appview/pages/templates/search/search.html @@ -1,7 +1,7 @@ -{{ define "title" }}Search · Tangled{{ end }} +{{ define "title" }}{{ if eq .FilterType "code" }}Code Search{{ else }}Search{{ end }} · Tangled{{ end }} {{ define "content" }} -

Search

+

{{ if eq .FilterType "code" }}Code Search{{ else }}Search{{ end }}

@@ -17,48 +17,89 @@ {{ define "searchBar" }}
+
- {{ i "x" "w-4 h-4" }} + {{ i "x" "size-4" }}
-
- {{ template "sortOptionsList" . }} -
+
-
-
- {{ template "languageFilters" . }} +
+
+

Filters

+ +
+
+ {{ template "searchTypeSwitcher" . }} + {{ if eq .FilterType "repo" }} +
+

Sort by

+ {{ template "sortOptionsList" . }} +
+ {{ end }} +
+

Languages

+ {{ template "languageFilters" . }} +
{{ end }} {{ define "searchResults" }} + {{ if .ErrorMsg }} +
+ {{ i "circle-alert" "size-4" }} + {{ .ErrorMsg }} +
+ {{ else if eq .FilterType "code" }} + {{ template "codeSearchResults" . }} + {{ else }} + {{ template "repoSearchResults" . }} + {{ end }} +{{ end }} + +{{ define "repoSearchResults" }}
{{ range .Repos }} -
- {{ template "user/fragments/repoCard" (list $ . true) }} -
+ {{ template "search/fragments/resultCard" . }} {{ else }}
No repositories found. @@ -75,19 +116,48 @@ "Page" .Page "TotalCount" .ResultCount "BasePath" "search" - "QueryParams" (queryParams "q" .FilterQuery "sort" .SortParam) + "QueryParams" (queryParams "q" .FilterQuery "sort" .SortParam "type" .FilterType) + ) }} + {{ end }} +{{ end }} + +{{ define "codeSearchResults" }} +
+ {{ range .Results }} + {{ template "search/fragments/resultCard" . }} + {{ else }} +
+ No results found. +
+ {{ end }} +
+ +
+ {{ template "searchStatistics" . }} +
+ + {{ if or (gt .Page.Offset 0) .HasMore }} + {{ template "fragments/pagination" (dict + "Page" .Page + "BasePath" "search" + "QueryParams" (queryParams "q" .FilterQuery "type" .FilterType) + "HasMore" .HasMore ) }} {{ end }} {{ end }} {{ define "searchOptionsPanel" }}
+ {{ template "searchTypeSwitcher" . }} + + {{ if eq .FilterType "repo" }}

Sort by

{{ template "sortOptionsList" . }}
+ {{ end }}

@@ -100,60 +170,87 @@

{{ end }} +{{ define "searchTypeSwitcher" }} +
+

+ Filter by +

+
+ {{ $options := list + (dict "value" "repo" "name" "Repositories") + (dict "value" "code" "name" "Code") + }} + {{ range $options }} + + {{ end }} +
+
+{{ end }} + {{ define "sortOptionsList" }} - {{ $currentQuery := .FilterQuery }} - + {{ $currentSort := .SortParam }} + {{ if eq $currentSort "" }} + {{ $currentSort = "relevance" }} + {{ end }} - {{ $options := list - (dict "value" "relevance" "name" "Relevance") - (dict "value" "created-desc" "name" "Newest") - (dict "value" "created-asc" "name" "Oldest") - (dict "value" "stars-desc" "name" "Most Stars") - (dict "value" "stars-asc" "name" "Fewest Stars") - (dict "value" "issues-desc" "name" "Most Issues") - (dict "value" "issues-asc" "name" "Fewest Issues") - (dict "value" "pulls-desc" "name" "Most Pulls") - (dict "value" "pulls-asc" "name" "Fewest Pulls") - }} - - {{ range $options }} - - {{ end }} - + {{ $options := list + (dict "value" "relevance" "name" "Relevance") + (dict "value" "created-desc" "name" "Newest") + (dict "value" "created-asc" "name" "Oldest") + (dict "value" "stars-desc" "name" "Most Stars") + (dict "value" "stars-asc" "name" "Fewest Stars") + (dict "value" "issues-desc" "name" "Most Issues") + (dict "value" "issues-asc" "name" "Fewest Issues") + (dict "value" "pulls-desc" "name" "Most Pulls") + (dict "value" "pulls-asc" "name" "Fewest Pulls") + }} + + {{ range $options }} + + {{ end }} + + {{ end }} {{ end }} {{ define "languageFilters" }} {{ $commonLanguages := list "Go" "JavaScript" "TypeScript" "Python" "Rust" "OCaml" "Haskell" "C" "C++" "Ruby" "Swift" }} -
+
{{ range $commonLanguages }} {{ $lang := . }} - {{ template "languageFilterChip" (dict "Language" $lang "CurrentQuery" $.FilterQuery) }} + {{ template "languageFilterChip" (dict "Language" $lang "CurrentQuery" $.FilterQuery "FilterType" $.FilterType) }} {{ end }}
{{ end }} {{ define "languageFilterChip" }} - {{ $lang := .Language }} + {{ $lang := .Language }} {{ $currentQuery := .CurrentQuery }} - {{ $langColor := langColor $lang }} - {{ $newQuery := queryParams "q" (printf "language:%s %s" $lang $currentQuery) }} + {{ $filterType := .FilterType }} + {{ $langColor := langColor $lang }} + {{ $newQuery := queryParams "q" (printf "lang:%s %s" $lang $currentQuery) "type" $filterType }} - Returned {{ .ResultCount }} of {{ .DocCount }} repos in {{ .TimeTaken }} -
+ {{ if eq .FilterType "code" }} + {{ if gt .MatchCount 0 }} +
+ Found {{ .MatchCount }} results in {{ .FileCount }} files in {{ .TimeTaken }} +
+ {{ end }} + {{ else }} + {{ if gt .ResultCount 0 }} +
+ Returned {{ .ResultCount }} of {{ .DocCount }} repos in {{ .TimeTaken }} +
+ {{ end }} {{ end }} {{ end }} - - diff --git a/appview/state/codesearch.go b/appview/state/codesearch.go new file mode 100644 index 00000000..5af78779 --- /dev/null +++ b/appview/state/codesearch.go @@ -0,0 +1,113 @@ +package state + +import ( + "errors" + "net/http" + "net/url" + "slices" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/appview/codesearch" + "tangled.org/core/appview/db" + "tangled.org/core/appview/models" + "tangled.org/core/appview/pages" + "tangled.org/core/appview/pagination" + "tangled.org/core/orm" +) + +func (s *State) handleCodeSearch(w http.ResponseWriter, r *http.Request) { + l := s.logger.With("handler", "CodeSearch") + ctx := r.Context() + page := pagination.FromContext(ctx) + q := r.URL.Query().Get("q") + + var redirected bool + var params pages.CodeSearchParams + params.BaseParams = pages.BaseParamsFromContext(ctx) + params.FilterQuery = q + params.Page = page + defer func() { + if redirected { + return + } + if err := s.pages.CodeSearch(w, params); err != nil { + l.Error("failed to render code search", "err", err) + } + }() + + if q == "" { + return + } + + res, err := s.codesearch.Search(ctx, q, page) + if err != nil { + // repo-name-only queries belong to the repo search page; redirect with + // the rewritten query (repo: prefix dropped, lang: kept). + var repoErr *codesearch.RepoOnlyError + if errors.As(err, &repoErr) { + redirected = true + http.Redirect(w, r, "/search?q="+url.QueryEscape(repoErr.Query), http.StatusFound) + return + } + l.Error("code search failed", "err", err, "query", q) + params.ErrorMsg = "Failed to perform search. Please try again later." + return + } + results := res.Results + + repoMap := map[syntax.DID]*models.Repo{} + var repoDids []string + for _, res := range results { + if res.RepoDID == "" { + continue + } + if _, ok := repoMap[res.RepoDID]; !ok { + repoMap[res.RepoDID] = nil + repoDids = append(repoDids, res.RepoDID.String()) + } + } + if len(repoDids) > 0 { + repos, err := db.GetRepos(s.db, orm.FilterIn("repo_did", repoDids)) + if err != nil { + l.Error("failed to load repos for code search", "err", err) + params.ErrorMsg = "Failed to load repos for code search. Please try again later." + return + } + for i := range repos { + repoMap[syntax.DID(repos[i].RepoDid)] = &repos[i] + } + } + + out := make([]pages.SearchResult, 0, len(results)) + for _, res := range results { + csr := pages.SearchResult{ + RepoDID: res.RepoDID, + Repo: repoMap[res.RepoDID], + FilePath: res.FilePath, + Branches: res.Branches, + Commit: res.Commit, + Language: res.Language, + } + if f := res.File; f != nil { + csr.File = &pages.CodeSearchResult_File{ + NameSpans: pages.FileNameSpans(res.FilePath, f.Ranges), + } + } + slices.SortStableFunc(res.Chunks, func(a, b models.Result_ChunkMatch) int { + return a.ContentStartLine - b.ContentStartLine + }) + for _, c := range res.Chunks { + csr.Chunks = append(csr.Chunks, pages.CodeSearchResult_Chunk{ + Lines: pages.ChunkLines(c.Content, c.ContentStartLine, c.Ranges), + MatchCount: len(c.Ranges), + }) + } + out = append(out, csr) + } + + params.Results = out + params.HasMore = res.HasMore + params.MatchCount = res.Stats.MatchCount + params.FileCount = res.Stats.FileCount + params.TimeTaken = res.Stats.Duration +} diff --git a/appview/state/search.go b/appview/state/search.go index 3b9be0be..de290132 100644 --- a/appview/state/search.go +++ b/appview/state/search.go @@ -17,22 +17,44 @@ import ( ) func (s *State) Search(w http.ResponseWriter, r *http.Request) { - l := s.logger.With("handler", "Search") + switch r.URL.Query().Get("type") { + case "code": + s.handleCodeSearch(w, r) + case "repo": + s.handleRepoSearch(w, r) + default: + query := r.URL.Query() + query.Set("type", "repo") + http.Redirect(w, r, "/search?"+query.Encode(), http.StatusFound) + } +} - params := r.URL.Query() +func (s *State) handleRepoSearch(w http.ResponseWriter, r *http.Request) { + l := s.logger.With("handler", "Search") + query := r.URL.Query() page := pagination.FromContext(r.Context()) + q := searchquery.Parse(query.Get("q")) - query := searchquery.Parse(params.Get("q")) - - sortParam := params.Get("sort") + sortParam := query.Get("sort") sortField, sortDesc := parseSortParam(sortParam) + var params pages.SearchReposParams + params.BaseParams = pages.BaseParamsFromContext(r.Context()) + params.FilterQuery = q.String() + params.SortParam = sortParam + params.Page = page + defer func() { + if err := s.pages.SearchRepos(w, params); err != nil { + l.Error("failed to render page", "err", err) + } + }() + var language string - if lang := cmp.Or(query.Get("language"), query.Get("lang")); lang != nil { + if lang := cmp.Or(q.Get("language"), q.Get("lang")); lang != nil { language = *lang } - tf := searchquery.ExtractTextFilters(query) + tf := searchquery.ExtractTextFilters(q) searchOpts := models.RepoSearchOptions{ Keywords: tf.Keywords, @@ -56,7 +78,7 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { res, err := s.indexer.Repos.Search(r.Context(), searchOpts) if err != nil { l.Error("failed to search repos", "err", err) - s.pages.Error500(w) + params.ErrorMsg = "Failed to perform search. Please try again later." return } @@ -66,7 +88,7 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { repos, err = db.GetRepos(s.db, orm.FilterIn("id", res.Hits)) if err != nil { l.Error("failed to get repos by IDs", "err", err) - s.pages.Error500(w) + params.ErrorMsg = "Failed to query repos. Please try again later." return } @@ -94,7 +116,7 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { ) if err != nil { l.Error("failed to get repos", "err", err) - s.pages.Error500(w) + params.ErrorMsg = "Failed to query repos. Please try again later." return } @@ -103,7 +125,7 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { ) if err != nil { l.Error("failed to count repos", "err", err) - s.pages.Error500(w) + params.ErrorMsg = "Failed to count repos. Please try again later." return } @@ -117,11 +139,11 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { "resultCount", resultCount, "docCount", docCount, "time", searchDuration, - "filterQuery", query.String(), + "filterQuery", q.String(), "sortParam", sortParam, ) - if !s.config.Core.Dev && query.String() != "" { + if !s.config.Core.Dev && q.String() != "" { distinctId := s.oauth.GetDid(r) if distinctId == "" { distinctId = "anonymous" @@ -131,7 +153,7 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { DistinctId: distinctId, Event: "search", Properties: posthog.Properties{ - "query": query.String(), + "query": q.String(), "result_count": resultCount, "method": method, }, @@ -141,19 +163,15 @@ func (s *State) Search(w http.ResponseWriter, r *http.Request) { }() } - err = s.pages.SearchRepos(w, pages.SearchReposParams{ - BaseParams: pages.BaseParamsFromContext(r.Context()), - Repos: repos, - Page: page, - FilterQuery: query.String(), - SortParam: sortParam, - TimeTaken: searchDuration, - ResultCount: resultCount, - DocCount: docCount, - }) - if err != nil { - l.Error("failed to render page", "err", err) + repoResults := make([]pages.SearchResult, len(repos)) + for i := range repos { + repoResults[i] = pages.SearchResult{Repo: &repos[i]} } + + params.Repos = repoResults + params.TimeTaken = searchDuration + params.ResultCount = resultCount + params.DocCount = docCount } func (s *State) SearchQuick(w http.ResponseWriter, r *http.Request) { diff --git a/appview/state/state.go b/appview/state/state.go index c391f45f..1172e3c7 100644 --- a/appview/state/state.go +++ b/appview/state/state.go @@ -15,6 +15,7 @@ import ( "tangled.org/core/appview/bsky" "tangled.org/core/appview/cache" "tangled.org/core/appview/cloudflare" + "tangled.org/core/appview/codesearch" "tangled.org/core/appview/config" "tangled.org/core/appview/db" "tangled.org/core/appview/email" @@ -76,6 +77,7 @@ type State struct { logger *slog.Logger validator *validator.Validator cfClient *cloudflare.Client + codesearch *codesearch.CodeSearch } func Make(ctx context.Context, config *config.Config) (*State, error) { @@ -250,6 +252,7 @@ func Make(ctx context.Context, config *config.Config) (*State, error) { logger: logger, validator: validator, cfClient: cfClient, + codesearch: &codesearch.CodeSearch{Host: config.CodeSearch.ZoektUrl}, } // fetch initial bluesky posts if configured diff --git a/docker-compose.yml b/docker-compose.yml index 4558b9f3..c6905efb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -338,6 +338,7 @@ services: TANGLED_JETSTREAM_ENDPOINT: wss://jetstream.tngl.boltless.dev/subscribe TANGLED_REDIS_ADDR: redis:6379 TANGLED_KNOTMIRROR_URL: https://mirror.tngl.boltless.dev + TANGLED_CODESEARCH_ZOEKT_URL: https://zoekt.tngl.boltless.dev ports: - "3000:3000" volumes: diff --git a/input.css b/input.css index d2427f09..7877eb89 100644 --- a/input.css +++ b/input.css @@ -554,6 +554,10 @@ @apply !bg-yellow-200/30 dark:!bg-yellow-700/30; } +.chunk-match-hl { + @apply rounded-sm !bg-yellow-300/70 dark:!bg-yellow-600/60; +} + :is(.line-quote-hl, .line-range-hl) > .min-w-\[3\.5rem\] { @apply !bg-yellow-200/30 dark:!bg-yellow-700/30; }