From 06a99c45da338bb47bbec2e408bf300593c4afad Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Sun, 22 Mar 2026 06:35:11 +0000 Subject: [PATCH] appview/indexer: add indexer for repos Signed-off-by: oppiliappan --- appview/indexer/indexer.go | 4 + appview/indexer/notifier.go | 9 + appview/indexer/repos/indexer.go | 372 +++++++++++++++ appview/indexer/repos/indexer_test.go | 639 ++++++++++++++++++++++++++ appview/models/search.go | 23 + 5 files changed, 1047 insertions(+) create mode 100644 appview/indexer/repos/indexer.go create mode 100644 appview/indexer/repos/indexer_test.go diff --git a/appview/indexer/indexer.go b/appview/indexer/indexer.go index 83d295f1..bea3ac27 100644 --- a/appview/indexer/indexer.go +++ b/appview/indexer/indexer.go @@ -7,6 +7,7 @@ import ( "tangled.org/core/appview/db" issues_indexer "tangled.org/core/appview/indexer/issues" pulls_indexer "tangled.org/core/appview/indexer/pulls" + repos_indexer "tangled.org/core/appview/indexer/repos" "tangled.org/core/appview/notify" tlog "tangled.org/core/log" ) @@ -14,6 +15,7 @@ import ( type Indexer struct { Issues *issues_indexer.Indexer Pulls *pulls_indexer.Indexer + Repos *repos_indexer.Indexer logger *slog.Logger notify.BaseNotifier } @@ -22,6 +24,7 @@ func New(logger *slog.Logger) *Indexer { return &Indexer{ issues_indexer.NewIndexer("indexes/issues.bleve"), pulls_indexer.NewIndexer("indexes/pulls.bleve"), + repos_indexer.NewIndexer("indexes/repos.bleve"), logger, notify.BaseNotifier{}, } @@ -32,5 +35,6 @@ func (ix *Indexer) Init(ctx context.Context, db *db.DB) error { ctx = tlog.IntoContext(ctx, ix.logger) ix.Issues.Init(ctx, db) ix.Pulls.Init(ctx, db) + ix.Repos.Init(ctx, db) return nil } diff --git a/appview/indexer/notifier.go b/appview/indexer/notifier.go index d848321e..e3c27c28 100644 --- a/appview/indexer/notifier.go +++ b/appview/indexer/notifier.go @@ -73,3 +73,12 @@ func (ix *Indexer) NewPullState(ctx context.Context, actor syntax.DID, pull *mod l.Error("failed to index a pr", "err", err) } } + +func (ix *Indexer) NewRepo(ctx context.Context, repo *models.Repo) { + l := log.FromContext(ctx).With("notifier", "indexer", "repo", repo) + l.Debug("indexing new repo") + err := ix.Repos.Index(ctx, *repo) + if err != nil { + l.Error("failed to index a repo", "err", err) + } +} diff --git a/appview/indexer/repos/indexer.go b/appview/indexer/repos/indexer.go new file mode 100644 index 00000000..41088300 --- /dev/null +++ b/appview/indexer/repos/indexer.go @@ -0,0 +1,372 @@ +// heavily inspired by gitea's model (basically copy-pasted) +package repos_indexer + +import ( + "context" + "errors" + "log" + "os" + + "github.com/blevesearch/bleve/v2" + "github.com/blevesearch/bleve/v2/analysis/analyzer/custom" + "github.com/blevesearch/bleve/v2/analysis/token/camelcase" + "github.com/blevesearch/bleve/v2/analysis/token/lowercase" + "github.com/blevesearch/bleve/v2/analysis/token/ngram" + "github.com/blevesearch/bleve/v2/analysis/token/unicodenorm" + "github.com/blevesearch/bleve/v2/analysis/tokenizer/unicode" + "github.com/blevesearch/bleve/v2/index/upsidedown" + "github.com/blevesearch/bleve/v2/mapping" + "github.com/blevesearch/bleve/v2/search/query" + "tangled.org/core/appview/db" + "tangled.org/core/appview/indexer/base36" + bleveutil "tangled.org/core/appview/indexer/bleve" + "tangled.org/core/appview/models" + "tangled.org/core/appview/pagination" + tlog "tangled.org/core/log" +) + +const ( + repoIndexerAnalyzer = "repoIndexer" + repoIndexerDocType = "repoIndexerDocType" + + unicodeNormalizeName = "unicodeNormalize" + + // Bump this when the index mapping changes to trigger a rebuild. + repoIndexerVersion = 5 +) + +type Indexer struct { + indexer bleve.Index + path string +} + +func NewIndexer(indexDir string) *Indexer { + return &Indexer{ + path: indexDir, + } +} + +// Init initializes the indexer +func (ix *Indexer) Init(ctx context.Context, e db.Execer) { + l := tlog.FromContext(ctx) + existed, err := ix.intialize(ctx) + if err != nil { + log.Fatalln("failed to initialize repo indexer", err) + } + if !existed { + l.Debug("Populating the repo indexer") + err := PopulateIndexer(ctx, ix, e) + if err != nil { + log.Fatalln("failed to populate repo indexer", err) + } + } + + count, _ := ix.indexer.DocCount() + l.Info("Initialized the repo indexer", "docCount", count) +} + +func generateRepoIndexMapping() (mapping.IndexMapping, error) { + mapping := bleve.NewIndexMapping() + docMapping := bleve.NewDocumentMapping() + + textFieldMapping := bleve.NewTextFieldMapping() + textFieldMapping.Store = false + textFieldMapping.IncludeInAll = false + + keywordFieldMapping := bleve.NewKeywordFieldMapping() + keywordFieldMapping.Store = false + keywordFieldMapping.IncludeInAll = false + + // case-insensitive keyword field for language and topics + caseInsensitiveKeywordMapping := bleve.NewTextFieldMapping() + caseInsensitiveKeywordMapping.Store = false + caseInsensitiveKeywordMapping.IncludeInAll = false + caseInsensitiveKeywordMapping.Analyzer = "keyword_lowercase" + + // trigram field for partial repo name matching + trigramFieldMapping := bleve.NewTextFieldMapping() + trigramFieldMapping.Store = false + trigramFieldMapping.IncludeInAll = false + trigramFieldMapping.Analyzer = "trigram" + + // text fields + docMapping.AddFieldMappingsAt("name", textFieldMapping) + docMapping.AddFieldMappingsAt("name_trigram", trigramFieldMapping) + docMapping.AddFieldMappingsAt("description", textFieldMapping) + docMapping.AddFieldMappingsAt("website", textFieldMapping) + docMapping.AddFieldMappingsAt("topics", textFieldMapping) + + // keyword fields + docMapping.AddFieldMappingsAt("language", caseInsensitiveKeywordMapping) + docMapping.AddFieldMappingsAt("topics_exact", caseInsensitiveKeywordMapping) + docMapping.AddFieldMappingsAt("did", keywordFieldMapping) + docMapping.AddFieldMappingsAt("knot", keywordFieldMapping) + docMapping.AddFieldMappingsAt("repo_at", keywordFieldMapping) + + err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ + "type": unicodenorm.Name, + "form": unicodenorm.NFC, + }) + if err != nil { + return nil, err + } + + err = mapping.AddCustomTokenFilter("edgeNgram3", map[string]any{ + "type": ngram.Name, + "min": 2.0, + "max": 3.0, + }) + if err != nil { + return nil, err + } + + err = mapping.AddCustomAnalyzer(repoIndexerAnalyzer, map[string]any{ + "type": custom.Name, + "char_filters": []string{}, + "tokenizer": unicode.Name, + "token_filters": []string{unicodeNormalizeName, camelcase.Name, lowercase.Name}, + }) + if err != nil { + return nil, err + } + + err = mapping.AddCustomAnalyzer("keyword_lowercase", map[string]any{ + "type": custom.Name, + "char_filters": []string{}, + "tokenizer": "single", + "token_filters": []string{lowercase.Name}, + }) + if err != nil { + return nil, err + } + + err = mapping.AddCustomAnalyzer("trigram", map[string]any{ + "type": custom.Name, + "char_filters": []string{}, + "tokenizer": "single", + "token_filters": []string{lowercase.Name, "edgeNgram3"}, + }) + if err != nil { + return nil, err + } + + mapping.DefaultAnalyzer = repoIndexerAnalyzer + mapping.AddDocumentMapping(repoIndexerDocType, docMapping) + mapping.AddDocumentMapping("_all", bleve.NewDocumentDisabledMapping()) + mapping.DefaultMapping = bleve.NewDocumentDisabledMapping() + + return mapping, nil +} + +func (ix *Indexer) intialize(ctx context.Context) (bool, error) { + if ix.indexer != nil { + return false, errors.New("indexer is already initialized") + } + + indexer, err := openIndexer(ctx, ix.path, repoIndexerVersion) + if err != nil { + return false, err + } + if indexer != nil { + ix.indexer = indexer + return true, nil + } + + mapping, err := generateRepoIndexMapping() + if err != nil { + return false, err + } + indexer, err = bleve.New(ix.path, mapping) + if err != nil { + return false, err + } + indexer.SetInternal([]byte("mapping_version"), []byte{byte(repoIndexerVersion)}) + + ix.indexer = indexer + + return false, nil +} + +func openIndexer(ctx context.Context, path string, version int) (bleve.Index, error) { + l := tlog.FromContext(ctx) + indexer, err := bleve.Open(path) + if err != nil { + if errors.Is(err, upsidedown.IncompatibleVersion) { + l.Info("Indexer was built with a previous version of bleve, deleting and rebuilding") + return nil, os.RemoveAll(path) + } + return nil, nil + } + + storedVersion, _ := indexer.GetInternal([]byte("mapping_version")) + if storedVersion == nil || int(storedVersion[0]) != version { + l.Info("Indexer mapping version changed, deleting and rebuilding") + indexer.Close() + return nil, os.RemoveAll(path) + } + + return indexer, nil +} + +func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error { + l := tlog.FromContext(ctx) + count := 0 + + err := pagination.IterateAll( + func(page pagination.Page) ([]models.Repo, error) { + return db.GetReposPaginated(e, page) + }, + func(repos []models.Repo) error { + count += len(repos) + return ix.Index(ctx, repos...) + }, + ) + + l.Info("repos indexed", "count", count) + return err +} + +type repoData struct { + ID int64 `json:"id"` + RepoAt string `json:"repo_at"` + Did string `json:"did"` + Name string `json:"name"` + NameTrigram string `json:"name_trigram"` + Description string `json:"description"` + Website string `json:"website"` + Topics []string `json:"topics"` + TopicsExact []string `json:"topics_exact"` + Knot string `json:"knot"` + Language string `json:"language"` +} + +func makeRepoData(repo *models.Repo) *repoData { + return &repoData{ + ID: repo.Id, + RepoAt: repo.RepoAt().String(), + Did: repo.Did, + Name: repo.Name, + NameTrigram: repo.Name, + Description: repo.Description, + Website: repo.Website, + Topics: repo.Topics, + TopicsExact: repo.Topics, + Knot: repo.Knot, + Language: repo.RepoStats.Language, + } +} + +// Type returns the document type, for bleve's mapping.Classifier interface. +func (r *repoData) Type() string { + return repoIndexerDocType +} + +type SearchResult struct { + Hits []int64 + Total uint64 +} + +const maxBatchSize = 20 + +func (ix *Indexer) Index(ctx context.Context, repos ...models.Repo) error { + batch := bleveutil.NewFlushingBatch(ix.indexer, maxBatchSize) + for _, repo := range repos { + repoData := makeRepoData(&repo) + if err := batch.Index(base36.Encode(repo.Id), repoData); err != nil { + return err + } + } + return batch.Flush() +} + +func (ix *Indexer) Delete(ctx context.Context, repoID int64) error { + return ix.indexer.Delete(base36.Encode(repoID)) +} + +func (ix *Indexer) Search(ctx context.Context, opts models.RepoSearchOptions) (*SearchResult, error) { + var musts []query.Query + var mustNots []query.Query + + for _, keyword := range opts.Keywords { + musts = append(musts, bleve.NewDisjunctionQuery( + bleveutil.MatchAndQuery("name", keyword, repoIndexerAnalyzer, 0), + bleveutil.MatchAndQuery("name_trigram", keyword, "trigram", 0), + bleveutil.MatchAndQuery("description", keyword, repoIndexerAnalyzer, 0), + bleveutil.MatchAndQuery("website", keyword, repoIndexerAnalyzer, 0), + bleveutil.MatchAndQuery("topics", keyword, repoIndexerAnalyzer, 0), + )) + } + + for _, phrase := range opts.Phrases { + musts = append(musts, bleve.NewDisjunctionQuery( + bleveutil.MatchPhraseQuery("name", phrase, repoIndexerAnalyzer), + bleveutil.MatchPhraseQuery("description", phrase, repoIndexerAnalyzer), + bleveutil.MatchPhraseQuery("website", phrase, repoIndexerAnalyzer), + bleveutil.MatchPhraseQuery("topics", phrase, repoIndexerAnalyzer), + )) + } + + for _, keyword := range opts.NegatedKeywords { + mustNots = append(mustNots, bleve.NewDisjunctionQuery( + bleveutil.MatchAndQuery("name", keyword, repoIndexerAnalyzer, 0), + bleveutil.MatchAndQuery("name_trigram", keyword, "trigram", 0), + bleveutil.MatchAndQuery("description", keyword, repoIndexerAnalyzer, 0), + bleveutil.MatchAndQuery("website", keyword, repoIndexerAnalyzer, 0), + bleveutil.MatchAndQuery("topics", keyword, repoIndexerAnalyzer, 0), + )) + } + + for _, phrase := range opts.NegatedPhrases { + mustNots = append(mustNots, bleve.NewDisjunctionQuery( + bleveutil.MatchPhraseQuery("name", phrase, repoIndexerAnalyzer), + bleveutil.MatchPhraseQuery("description", phrase, repoIndexerAnalyzer), + bleveutil.MatchPhraseQuery("website", phrase, repoIndexerAnalyzer), + bleveutil.MatchPhraseQuery("topics", phrase, repoIndexerAnalyzer), + )) + } + + // keyword filters + if opts.Language != "" { + musts = append(musts, bleveutil.MatchAndQuery("language", opts.Language, "keyword_lowercase", 0)) + } + + if opts.Knot != "" { + musts = append(musts, bleveutil.KeywordFieldQuery("knot", opts.Knot)) + } + + if opts.Did != "" { + musts = append(musts, bleveutil.KeywordFieldQuery("did", opts.Did)) + } + + for _, topic := range opts.Topics { + musts = append(musts, bleveutil.MatchAndQuery("topics_exact", topic, "keyword_lowercase", 0)) + } + + for _, topic := range opts.NegatedTopics { + mustNots = append(mustNots, bleveutil.MatchAndQuery("topics_exact", topic, "keyword_lowercase", 0)) + } + + indexerQuery := bleve.NewBooleanQuery() + if len(musts) == 0 { + musts = append(musts, bleve.NewMatchAllQuery()) + } + indexerQuery.AddMust(musts...) + indexerQuery.AddMustNot(mustNots...) + searchReq := bleve.NewSearchRequestOptions(indexerQuery, opts.Page.Limit, opts.Page.Offset, false) + res, err := ix.indexer.SearchInContext(ctx, searchReq) + if err != nil { + return nil, nil + } + ret := &SearchResult{ + Total: res.Total, + Hits: make([]int64, len(res.Hits)), + } + for i, hit := range res.Hits { + id, err := base36.Decode(hit.ID) + if err != nil { + return nil, err + } + ret.Hits[i] = id + } + return ret, nil +} diff --git a/appview/indexer/repos/indexer_test.go b/appview/indexer/repos/indexer_test.go new file mode 100644 index 00000000..5a8a686a --- /dev/null +++ b/appview/indexer/repos/indexer_test.go @@ -0,0 +1,639 @@ +package repos_indexer + +import ( + "context" + "os" + "testing" + + "github.com/blevesearch/bleve/v2" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "tangled.org/core/appview/models" + "tangled.org/core/appview/pagination" +) + +func setupTestIndexer(t *testing.T) (*Indexer, func()) { + t.Helper() + + tmpDir, err := os.MkdirTemp("", "repo_indexer_test") + require.NoError(t, err) + + ix := NewIndexer(tmpDir) + + mapping, err := generateRepoIndexMapping() + require.NoError(t, err) + + indexer, err := bleve.New(tmpDir, mapping) + require.NoError(t, err) + ix.indexer = indexer + + cleanup := func() { + ix.indexer.Close() + os.RemoveAll(tmpDir) + } + + return ix, cleanup +} + +func TestBasicIndexingAndSearch(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "web-framework", + Knot: "example.com", + Description: "A modern web framework for Go", + Website: "https://example.com/web-framework", + Topics: []string{"web", "framework", "golang"}, + RepoStats: &models.RepoStats{Language: "Go"}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "cli-tool", + Knot: "example.com", + Description: "Command line utility for developers", + Website: "", + Topics: []string{"cli", "tool"}, + RepoStats: &models.RepoStats{Language: "Rust"}, + }, + models.Repo{ + Id: 3, + Did: "did:plc:alice", + Name: "javascript-parser", + Knot: "example.com", + Description: "Fast JavaScript parser", + Website: "", + Topics: []string{"javascript", "parser"}, + RepoStats: &models.RepoStats{Language: "JavaScript"}, + }, + ) + require.NoError(t, err) + + // search by name + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"framework"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) + + // search by description + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"utility"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(2)) + + // search by website + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"example.com/web-framework"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) +} + +func TestLanguageFiltering(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "go-project", + RepoStats: &models.RepoStats{Language: "Go"}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "rust-project", + RepoStats: &models.RepoStats{Language: "Rust"}, + }, + models.Repo{ + Id: 3, + Did: "did:plc:alice", + Name: "another-go-project", + RepoStats: &models.RepoStats{Language: "Go"}, + }, + ) + require.NoError(t, err) + + // filter by go language + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Language: "Go", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) + + // filter by rust language + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Language: "Rust", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(2)) +} + +func TestTopicExactMatching(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "js-tool", + Topics: []string{"javascript", "tool"}, + RepoStats: &models.RepoStats{}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "java-app", + Topics: []string{"java", "application"}, + RepoStats: &models.RepoStats{}, + }, + models.Repo{ + Id: 3, + Did: "did:plc:alice", + Name: "cli-tool", + Topics: []string{"cli", "tool"}, + RepoStats: &models.RepoStats{}, + }, + ) + require.NoError(t, err) + + // exact match for "javascript" topic + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Topics: []string{"javascript"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) + + // exact match for "tool" topic (should match repos 1 and 3) + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Topics: []string{"tool"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) +} + +func TestTopicTextSearch(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "js-tool", + Topics: []string{"JavaScript"}, + RepoStats: &models.RepoStats{}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "java-app", + Topics: []string{"Java"}, + RepoStats: &models.RepoStats{}, + }, + ) + require.NoError(t, err) + + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"Java"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(2)) +} + +func TestNegatedFilters(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "active-project", + Description: "An active development project", + Topics: []string{"active"}, + RepoStats: &models.RepoStats{Language: "Go"}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "archived-project", + Description: "An archived project", + Topics: []string{"archived"}, + RepoStats: &models.RepoStats{Language: "Python"}, + }, + models.Repo{ + Id: 3, + Did: "did:plc:alice", + Name: "another-project", + Description: "Another active project", + Topics: []string{"active"}, + RepoStats: &models.RepoStats{Language: "Go"}, + }, + ) + require.NoError(t, err) + + // exclude archived topic + result, err := ix.Search(ctx, models.RepoSearchOptions{ + NegatedTopics: []string{"archived"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) + + // exclude keyword "archived" + result, err = ix.Search(ctx, models.RepoSearchOptions{ + NegatedKeywords: []string{"archived"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) + + // exclude phrase + result, err = ix.Search(ctx, models.RepoSearchOptions{ + NegatedPhrases: []string{"archived project"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) +} + +func TestPagination(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + // index multiple repos + var repos []models.Repo + for i := 1; i <= 25; i++ { + repos = append(repos, models.Repo{ + Id: int64(i), + Did: "did:plc:alice", + Name: "project", + Topics: []string{"test"}, + RepoStats: &models.RepoStats{}, + }) + } + err := ix.Index(ctx, repos...) + require.NoError(t, err) + + // first page + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Topics: []string{"test"}, + Page: pagination.Page{Limit: 10, Offset: 0}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(25), result.Total) + assert.Len(t, result.Hits, 10) + + // second page + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Topics: []string{"test"}, + Page: pagination.Page{Limit: 10, Offset: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(25), result.Total) + assert.Len(t, result.Hits, 10) + + // third page - 5 items + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Topics: []string{"test"}, + Page: pagination.Page{Limit: 10, Offset: 20}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(25), result.Total) + assert.Len(t, result.Hits, 5) +} + +func TestUpdateReindex(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + // initial index + err := ix.Index(ctx, models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "my-project", + Description: "Initial description", + Topics: []string{"initial"}, + RepoStats: &models.RepoStats{Language: "Go"}, + }) + require.NoError(t, err) + + // search for initial state + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"Initial"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + + // update the repo + err = ix.Index(ctx, models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "my-project", + Description: "Updated description", + Topics: []string{"updated"}, + RepoStats: &models.RepoStats{Language: "Rust"}, + }) + require.NoError(t, err) + + // search for old description should return nothing + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"Initial"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(0), result.Total) + + // search for new description should work + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"Updated"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + + // language should be updated + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Language: "Rust", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) +} + +func TestEmptyResults(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "my-project", + RepoStats: &models.RepoStats{}, + }) + require.NoError(t, err) + + // search for non-existent keyword + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"nonexistent"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(0), result.Total) + assert.Empty(t, result.Hits) + + // search for non-existent language + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Language: "NonexistentLanguage", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(0), result.Total) + assert.Empty(t, result.Hits) +} + +func TestCombinedFilters(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "web-server", + Knot: "example.com", + Description: "A web server in Go", + Topics: []string{"web", "server"}, + RepoStats: &models.RepoStats{Language: "Go"}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "web-client", + Knot: "example.org", + Description: "A web client in Rust", + Topics: []string{"web", "client"}, + RepoStats: &models.RepoStats{Language: "Rust"}, + }, + models.Repo{ + Id: 3, + Did: "did:plc:alice", + Name: "cli-tool", + Knot: "example.com", + Description: "A CLI tool in Go", + Topics: []string{"cli", "tool"}, + RepoStats: &models.RepoStats{Language: "Go"}, + }, + ) + require.NoError(t, err) + + // combine language + topic + keyword + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Language: "Go", + Topics: []string{"web"}, + Keywords: []string{"server"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) + + // combine did + language + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Did: "did:plc:alice", + Language: "Go", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) + + // combine knot + language + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Knot: "example.com", + Language: "Go", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) +} + +func TestRepoWithoutLanguage(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "project-with-language", + RepoStats: &models.RepoStats{Language: "Go"}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "project-without-language", + RepoStats: &models.RepoStats{Language: ""}, + }, + ) + require.NoError(t, err) + + // search without language filter should return both + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"project"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + + // language filter should only return repo with language + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Language: "Go", + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) +} + +func TestRepoWithoutTopics(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "project-with-topics", + Topics: []string{"cli", "tool"}, + RepoStats: &models.RepoStats{}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "project-without-topics", + Topics: []string{}, + RepoStats: &models.RepoStats{}, + }, + ) + require.NoError(t, err) + + // topic filter should only return repo with topics + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Topics: []string{"cli"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) + + // general search should return both + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Keywords: []string{"project"}, + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) +} + +func TestDelete(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Repo{ + Id: 1, + Did: "did:plc:alice", + Name: "to-delete", + RepoStats: &models.RepoStats{}, + }, + models.Repo{ + Id: 2, + Did: "did:plc:bob", + Name: "to-keep", + RepoStats: &models.RepoStats{}, + }, + ) + require.NoError(t, err) + + // verify both exist + result, err := ix.Search(ctx, models.RepoSearchOptions{ + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + + // delete repo 1 + err = ix.Delete(ctx, 1) + require.NoError(t, err) + + // verify only one remains + result, err = ix.Search(ctx, models.RepoSearchOptions{ + Page: pagination.Page{Limit: 10}, + }) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(2)) +} diff --git a/appview/models/search.go b/appview/models/search.go index c630193d..ac06e03c 100644 --- a/appview/models/search.go +++ b/appview/models/search.go @@ -53,3 +53,26 @@ func (o *PullSearchOptions) HasSearchFilters() bool { len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 || len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 } + +type RepoSearchOptions struct { + Keywords []string // text search across name, description, website, topics + Phrases []string // phrase search + + Language string // exact match on primary language + Knot string // filter by knot domain + Did string // filter by owner DID + Topics []string // exact topic matches + + NegatedKeywords []string + NegatedPhrases []string + NegatedTopics []string + + Page pagination.Page +} + +func (o *RepoSearchOptions) HasSearchFilters() bool { + return len(o.Keywords) > 0 || len(o.Phrases) > 0 || + o.Language != "" || o.Did != "" || + len(o.Topics) > 0 || len(o.NegatedTopics) > 0 || + len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 +} -- 2.51.2