diff --git a/appview/indexer/issues/indexer.go b/appview/indexer/issues/indexer.go index ceb64efe..d5a13425 100644 --- a/appview/indexer/issues/indexer.go +++ b/appview/indexer/issues/indexer.go @@ -18,7 +18,7 @@ import ( "github.com/blevesearch/bleve/v2/search/query" "tangled.org/core/appview/db" "tangled.org/core/appview/indexer/base36" - "tangled.org/core/appview/indexer/bleve" + bleveutil "tangled.org/core/appview/indexer/bleve" "tangled.org/core/appview/models" "tangled.org/core/appview/pagination" tlog "tangled.org/core/log" @@ -31,7 +31,7 @@ const ( unicodeNormalizeName = "uicodeNormalize" // Bump this when the index mapping changes to trigger a rebuild. - issueIndexerVersion = 2 + issueIndexerVersion = 3 ) type Indexer struct { @@ -89,6 +89,7 @@ func generateIssueIndexMapping() (mapping.IndexMapping, error) { docMapping.AddFieldMappingsAt("is_open", boolFieldMapping) docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping) docMapping.AddFieldMappingsAt("labels", keywordFieldMapping) + docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping) err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ "type": unicodenorm.Name, @@ -183,28 +184,30 @@ func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error { } type issueData struct { - ID int64 `json:"id"` - RepoAt string `json:"repo_at"` - IssueID int `json:"issue_id"` - Title string `json:"title"` - Body string `json:"body"` - IsOpen bool `json:"is_open"` - AuthorDid string `json:"author_did"` - Labels []string `json:"labels"` + ID int64 `json:"id"` + RepoAt string `json:"repo_at"` + IssueID int `json:"issue_id"` + Title string `json:"title"` + Body string `json:"body"` + IsOpen bool `json:"is_open"` + AuthorDid string `json:"author_did"` + Labels []string `json:"labels"` + LabelValues []string `json:"label_values"` Comments []IssueCommentData `json:"comments"` } func makeIssueData(issue *models.Issue) *issueData { return &issueData{ - ID: issue.Id, - RepoAt: issue.RepoAt.String(), - IssueID: issue.IssueId, - Title: issue.Title, - Body: issue.Body, - IsOpen: issue.Open, - AuthorDid: issue.Did, - Labels: issue.Labels.LabelNames(), + ID: issue.Id, + RepoAt: issue.RepoAt.String(), + IssueID: issue.IssueId, + Title: issue.Title, + Body: issue.Body, + IsOpen: issue.Open, + AuthorDid: issue.Did, + Labels: issue.Labels.LabelNames(), + LabelValues: issue.Labels.LabelNameValues(), } } @@ -284,14 +287,22 @@ func (ix *Indexer) Search(ctx context.Context, opts models.IssueSearchOptions) ( musts = append(musts, bleveutil.KeywordFieldQuery("labels", label)) } - if opts.NegatedAuthorDid != "" { - mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid)) + for _, did := range opts.NegatedAuthorDids { + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did)) } for _, label := range opts.NegatedLabels { mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label)) } + for _, lv := range opts.LabelValues { + musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv)) + } + + for _, lv := range opts.NegatedLabelValues { + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv)) + } + indexerQuery := bleve.NewBooleanQuery() indexerQuery.AddMust(musts...) indexerQuery.AddMustNot(mustNots...) diff --git a/appview/indexer/issues/indexer_test.go b/appview/indexer/issues/indexer_test.go index da974e16..93655d79 100644 --- a/appview/indexer/issues/indexer_test.go +++ b/appview/indexer/issues/indexer_test.go @@ -3,6 +3,7 @@ package issues_indexer import ( "context" "os" + "strings" "testing" "github.com/blevesearch/bleve/v2" @@ -38,11 +39,23 @@ func setupTestIndexer(t *testing.T) (*Indexer, func()) { func boolPtr(b bool) *bool { return &b } -func makeLabelState(labels ...string) models.LabelState { +// makeLabelState creates a LabelState for testing. Each entry is either +// "name" (null-type label) or "name=val" (valued label). +func makeLabelState(entries ...string) models.LabelState { state := models.NewLabelState() - for _, label := range labels { - state.Inner()[label] = make(map[string]struct{}) - state.SetName(label, label) + for _, entry := range entries { + if eqIdx := strings.Index(entry, "="); eqIdx > 0 { + name := entry[:eqIdx] + val := entry[eqIdx+1:] + if state.Inner()[name] == nil { + state.Inner()[name] = make(map[string]struct{}) + } + state.Inner()[name][val] = struct{}{} + state.SetName(name, name) + } else { + state.Inner()[entry] = make(map[string]struct{}) + state.SetName(entry, entry) + } } return state } @@ -262,3 +275,70 @@ func TestSearchNoResults(t *testing.T) { assert.Equal(t, uint64(0), result.Total) assert.Empty(t, result.Hits) } + +func TestSearchLabelValues(t *testing.T) { + ix, cleanup := setupTestIndexer(t) + defer cleanup() + + ctx := context.Background() + + err := ix.Index(ctx, + models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority bug", Body: "Urgent", Open: true, Did: "did:plc:alice", + Labels: makeLabelState("bug", "priority=high")}, + models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Low priority feature", Body: "Nice to have", Open: true, Did: "did:plc:bob", + Labels: makeLabelState("feature", "priority=low")}, + models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority feature", Body: "Important", Open: true, Did: "did:plc:alice", + Labels: makeLabelState("feature", "priority=high")}, + ) + require.NoError(t, err) + + opts := func() models.IssueSearchOptions { + return models.IssueSearchOptions{ + RepoAt: "at://did:plc:test/sh.tangled.repo/abc", + IsOpen: boolPtr(true), + Page: pagination.Page{Limit: 10}, + } + } + + o := opts() + o.LabelValues = []string{"priority:high"} + result, err := ix.Search(ctx, o) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) + + o = opts() + o.LabelValues = []string{"priority:low"} + result, err = ix.Search(ctx, o) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(2)) + + // Combined: plain label + label value + o = opts() + o.Labels = []string{"feature"} + o.LabelValues = []string{"priority:high"} + result, err = ix.Search(ctx, o) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(3)) + + // Negated label value + o = opts() + o.NegatedLabelValues = []string{"priority:low"} + result, err = ix.Search(ctx, o) + require.NoError(t, err) + assert.Equal(t, uint64(2), result.Total) + assert.Contains(t, result.Hits, int64(1)) + assert.Contains(t, result.Hits, int64(3)) + + // Label value + negated plain label + o = opts() + o.LabelValues = []string{"priority:high"} + o.NegatedLabels = []string{"feature"} + result, err = ix.Search(ctx, o) + require.NoError(t, err) + assert.Equal(t, uint64(1), result.Total) + assert.Contains(t, result.Hits, int64(1)) +} diff --git a/appview/indexer/pulls/indexer.go b/appview/indexer/pulls/indexer.go index 5481993e..ed225f7a 100644 --- a/appview/indexer/pulls/indexer.go +++ b/appview/indexer/pulls/indexer.go @@ -18,7 +18,7 @@ import ( "github.com/blevesearch/bleve/v2/search/query" "tangled.org/core/appview/db" "tangled.org/core/appview/indexer/base36" - "tangled.org/core/appview/indexer/bleve" + bleveutil "tangled.org/core/appview/indexer/bleve" "tangled.org/core/appview/models" tlog "tangled.org/core/log" ) @@ -30,7 +30,7 @@ const ( unicodeNormalizeName = "uicodeNormalize" // Bump this when the index mapping changes to trigger a rebuild. - pullIndexerVersion = 2 + pullIndexerVersion = 3 ) type Indexer struct { @@ -84,6 +84,7 @@ func generatePullIndexMapping() (mapping.IndexMapping, error) { docMapping.AddFieldMappingsAt("state", keywordFieldMapping) docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping) docMapping.AddFieldMappingsAt("labels", keywordFieldMapping) + docMapping.AddFieldMappingsAt("label_values", keywordFieldMapping) err := mapping.AddCustomTokenFilter(unicodeNormalizeName, map[string]any{ "type": unicodenorm.Name, @@ -178,28 +179,30 @@ func PopulateIndexer(ctx context.Context, ix *Indexer, e db.Execer) error { } type pullData struct { - ID int64 `json:"id"` - RepoAt string `json:"repo_at"` - PullID int `json:"pull_id"` - Title string `json:"title"` - Body string `json:"body"` - State string `json:"state"` - AuthorDid string `json:"author_did"` - Labels []string `json:"labels"` + ID int64 `json:"id"` + RepoAt string `json:"repo_at"` + PullID int `json:"pull_id"` + Title string `json:"title"` + Body string `json:"body"` + State string `json:"state"` + AuthorDid string `json:"author_did"` + Labels []string `json:"labels"` + LabelValues []string `json:"label_values"` Comments []pullCommentData `json:"comments"` } func makePullData(pull *models.Pull) *pullData { return &pullData{ - ID: int64(pull.ID), - RepoAt: pull.RepoAt.String(), - PullID: pull.PullId, - Title: pull.Title, - Body: pull.Body, - State: pull.State.String(), - AuthorDid: pull.OwnerDid, - Labels: pull.Labels.LabelNames(), + ID: int64(pull.ID), + RepoAt: pull.RepoAt.String(), + PullID: pull.PullId, + Title: pull.Title, + Body: pull.Body, + State: pull.State.String(), + AuthorDid: pull.OwnerDid, + Labels: pull.Labels.LabelNames(), + LabelValues: pull.Labels.LabelNameValues(), } } @@ -285,14 +288,22 @@ func (ix *Indexer) Search(ctx context.Context, opts models.PullSearchOptions) (* musts = append(musts, bleveutil.KeywordFieldQuery("labels", label)) } - if opts.NegatedAuthorDid != "" { - mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", opts.NegatedAuthorDid)) + for _, did := range opts.NegatedAuthorDids { + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("author_did", did)) } for _, label := range opts.NegatedLabels { mustNots = append(mustNots, bleveutil.KeywordFieldQuery("labels", label)) } + for _, lv := range opts.LabelValues { + musts = append(musts, bleveutil.KeywordFieldQuery("label_values", lv)) + } + + for _, lv := range opts.NegatedLabelValues { + mustNots = append(mustNots, bleveutil.KeywordFieldQuery("label_values", lv)) + } + indexerQuery := bleve.NewBooleanQuery() indexerQuery.AddMust(musts...) indexerQuery.AddMustNot(mustNots...) diff --git a/appview/models/label.go b/appview/models/label.go index d851de2c..13e1e988 100644 --- a/appview/models/label.go +++ b/appview/models/label.go @@ -314,6 +314,27 @@ func (s LabelState) LabelNames() []string { return result } +// LabelNameValues returns composite "name:value" strings for all labels +// that have non-empty values. +func (s LabelState) LabelNameValues() []string { + var result []string + for key, valset := range s.inner { + if valset == nil { + continue + } + name, ok := s.names[key] + if !ok { + continue + } + for val := range valset { + if val != "" { + result = append(result, name+":"+val) + } + } + } + return result +} + func (s LabelState) Inner() map[string]set { return s.inner } diff --git a/appview/models/search.go b/appview/models/search.go index ea0d0d95..c630193d 100644 --- a/appview/models/search.go +++ b/appview/models/search.go @@ -3,47 +3,53 @@ package models import "tangled.org/core/appview/pagination" type IssueSearchOptions struct { - Keywords []string - Phrases []string - RepoAt string - IsOpen *bool - AuthorDid string - Labels []string - - NegatedKeywords []string - NegatedPhrases []string - NegatedLabels []string - NegatedAuthorDid string + Keywords []string + Phrases []string + RepoAt string + IsOpen *bool + AuthorDid string + Labels []string + LabelValues []string + + NegatedKeywords []string + NegatedPhrases []string + NegatedLabels []string + NegatedLabelValues []string + NegatedAuthorDids []string Page pagination.Page } func (o *IssueSearchOptions) HasSearchFilters() bool { return len(o.Keywords) > 0 || len(o.Phrases) > 0 || - o.AuthorDid != "" || o.NegatedAuthorDid != "" || + o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 || len(o.Labels) > 0 || len(o.NegatedLabels) > 0 || + len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 || len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 } type PullSearchOptions struct { - Keywords []string - Phrases []string - RepoAt string - State *PullState - AuthorDid string - Labels []string - - NegatedKeywords []string - NegatedPhrases []string - NegatedLabels []string - NegatedAuthorDid string + Keywords []string + Phrases []string + RepoAt string + State *PullState + AuthorDid string + Labels []string + LabelValues []string + + NegatedKeywords []string + NegatedPhrases []string + NegatedLabels []string + NegatedLabelValues []string + NegatedAuthorDids []string Page pagination.Page } func (o *PullSearchOptions) HasSearchFilters() bool { return len(o.Keywords) > 0 || len(o.Phrases) > 0 || - o.AuthorDid != "" || o.NegatedAuthorDid != "" || + o.AuthorDid != "" || len(o.NegatedAuthorDids) > 0 || len(o.Labels) > 0 || len(o.NegatedLabels) > 0 || + len(o.LabelValues) > 0 || len(o.NegatedLabelValues) > 0 || len(o.NegatedKeywords) > 0 || len(o.NegatedPhrases) > 0 }