text search for the atmosphere
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103package classify
import ( "strings" "unicode/utf8")
// Classifier decides which records to index and pulls prose text out of them.// Construct via New with values loaded from config.type Classifier struct { blockExact map[string]struct{} blockPrefix []string proseFields map[string]struct{} minRuneLen int}
func New(blocklist, proseFields []string, minRuneLen int) *Classifier { c := &Classifier{ blockExact: make(map[string]struct{}), proseFields: make(map[string]struct{}, len(proseFields)), minRuneLen: minRuneLen, } for _, pat := range blocklist { if strings.HasSuffix(pat, ".*") { // Trailing '.' keeps "app.bsky.*" from also blocking "app.bskyish". c.blockPrefix = append(c.blockPrefix, strings.TrimSuffix(pat, "*")) } else { c.blockExact[pat] = struct{}{} } } for _, f := range proseFields { c.proseFields[f] = struct{}{} } if c.minRuneLen < 1 { c.minRuneLen = 1 } return c}
// IsBlocked reports whether nsid matches any exact entry or prefix pattern// (e.g. "app.bsky.*") in the blocklist.func (c *Classifier) IsBlocked(nsid string) bool { if _, ok := c.blockExact[nsid]; ok { return true } for _, p := range c.blockPrefix { if strings.HasPrefix(nsid, p) { return true } } return false}
// Extracted is what the indexer stores per record.type Extracted struct { // Text is a newline-joined concatenation of every prose field we found. // Empty if the record carried no indexable prose. Text string // CreatedAt is the record's createdAt string, if present. Left unparsed // so Meili can filter/sort as ISO-8601 lex order. CreatedAt string // Lang is the first language tag we found (langs[0]), if any. Lang string}
// Extract walks a decoded record (map[string]any from Jetstream) and pulls// out prose-shaped fields plus a couple of common metadata fields.func (c *Classifier) Extract(record map[string]any) Extracted { var out Extracted var parts []string c.walk(record, &parts) out.Text = strings.Join(parts, "\n")
if v, ok := record["createdAt"].(string); ok { out.CreatedAt = v } if langs, ok := record["langs"].([]any); ok && len(langs) > 0 { if s, ok := langs[0].(string); ok { out.Lang = s } } return out}
func (c *Classifier) walk(v any, out *[]string) { switch x := v.(type) { case map[string]any: for k, child := range x { if _, ok := c.proseFields[k]; ok { if s, ok := child.(string); ok && utf8.RuneCountInString(s) >= c.minRuneLen { *out = append(*out, s) continue } } c.walk(child, out) } case []any: for _, item := range x { c.walk(item, out) } }}