package classify import ( "strings" "unicode/utf8" ) // Classifier decides which records to index and pulls prose text out of them. // Construct via New with values loaded from config. type Classifier struct { blockExact map[string]struct{} blockPrefix []string proseFields map[string]struct{} minRuneLen int } func New(blocklist, proseFields []string, minRuneLen int) *Classifier { c := &Classifier{ blockExact: make(map[string]struct{}), proseFields: make(map[string]struct{}, len(proseFields)), minRuneLen: minRuneLen, } for _, pat := range blocklist { if strings.HasSuffix(pat, ".*") { // Trailing '.' keeps "app.bsky.*" from also blocking "app.bskyish". c.blockPrefix = append(c.blockPrefix, strings.TrimSuffix(pat, "*")) } else { c.blockExact[pat] = struct{}{} } } for _, f := range proseFields { c.proseFields[f] = struct{}{} } if c.minRuneLen < 1 { c.minRuneLen = 1 } return c } // IsBlocked reports whether nsid matches any exact entry or prefix pattern // (e.g. "app.bsky.*") in the blocklist. func (c *Classifier) IsBlocked(nsid string) bool { if _, ok := c.blockExact[nsid]; ok { return true } for _, p := range c.blockPrefix { if strings.HasPrefix(nsid, p) { return true } } return false } // Extracted is what the indexer stores per record. type Extracted struct { // Text is a newline-joined concatenation of every prose field we found. // Empty if the record carried no indexable prose. Text string // CreatedAt is the record's createdAt string, if present. Left unparsed // so Meili can filter/sort as ISO-8601 lex order. CreatedAt string // Lang is the first language tag we found (langs[0]), if any. Lang string } // Extract walks a decoded record (map[string]any from Jetstream) and pulls // out prose-shaped fields plus a couple of common metadata fields. func (c *Classifier) Extract(record map[string]any) Extracted { var out Extracted var parts []string c.walk(record, &parts) out.Text = strings.Join(parts, "\n") if v, ok := record["createdAt"].(string); ok { out.CreatedAt = v } if langs, ok := record["langs"].([]any); ok && len(langs) > 0 { if s, ok := langs[0].(string); ok { out.Lang = s } } return out } func (c *Classifier) walk(v any, out *[]string) { switch x := v.(type) { case map[string]any: for k, child := range x { if _, ok := c.proseFields[k]; ok { if s, ok := child.(string); ok && utf8.RuneCountInString(s) >= c.minRuneLen { *out = append(*out, s) continue } } c.walk(child, out) } case []any: for _, item := range x { c.walk(item, out) } } }