From bdebc315e4cb2f435179f50905147f3e908560e9 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Thu, 11 Jun 2026 06:50:38 +0000 Subject: [PATCH] appview/db: speed up following timeline queries benchmarked on a synthetic db (10k users, 500k follows, 500k stars, 50k repos), the per-source queries behind the following timeline were slow because of missing indexes and a poor query shape: - stars filtered by did had no usable index; sqlite picked idx_stars_subject_type which matches nearly every row (~312ms) - follower/following counts group by subject_did, which had no index, forcing a full scan (~229ms) - follows ordered by followed_at with no index sorted every matched row in a temp b-tree (~157ms) fixes: - new indexes: stars(did, subject_type, created), follows(subject_did), follows(followed_at), repos(created); the latter two also let the global timeline read newest-first straight off an index instead of sorting - push the following-set membership check into sql with a subquery (orm.FilterInSubquery) instead of materializing all followed dids in go and binding them as hundreds of placeholders in three separate queries all benchmarked queries now run in ~1ms. Signed-off-by: Anirudh Oppiliappan --- orm/orm.go | 20 ++++++++++++++++++++ appview/db/db.go | 14 ++++++++++++++ appview/db/timeline.go | 49 ++++++++++++++++++++++++------------------------- appview/db/timeline_test.go | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 file(s) changed, 183 insertion(s)(+), 25 deletion(s)(-) diff --git a/orm/orm.go b/orm/orm.go --- a/orm/orm.go +++ b/orm/orm.go @@ -93,7 +93,23 @@ return newFilter(key, "like", fmt.Sprintf("%%%v%%", arg)) } +// FilterInSubquery compiles to `key in (subquery)`, binding args within the +// subquery. Prefer this over FilterIn with a large materialized list: it +// keeps the query text constant and lets sqlite plan a semi-join. +func FilterInSubquery(key, subquery string, args ...any) Filter { + return newFilter(key, "in", subqueryArg{query: subquery, args: args}) +} + +type subqueryArg struct { + query string + args []any +} + func (f Filter) Condition() string { + if sub, ok := f.arg.(subqueryArg); ok { + return fmt.Sprintf("%s %s (%s)", f.Key, f.Cmp, sub.query) + } + rv := reflect.ValueOf(f.arg) kind := rv.Kind() @@ -116,6 +132,10 @@ } func (f Filter) Arg() []any { + if sub, ok := f.arg.(subqueryArg); ok { + return sub.args + } + rv := reflect.ValueOf(f.arg) kind := rv.Kind() if (kind == reflect.Slice && rv.Type().Elem().Kind() != reflect.Uint8) || kind == reflect.Array { diff --git a/appview/db/db.go b/appview/db/db.go --- a/appview/db/db.go +++ b/appview/db/db.go @@ -2178,6 +2178,20 @@ return err }) + orm.RunMigration(conn, logger, "timeline-query-indexes", func(tx *sql.Tx) error { + _, err := tx.Exec(` + -- following timeline: stars by a set of users, newest first + create index if not exists idx_stars_did_type_created on stars(did, subject_type, created); + -- follower counts and reverse lookups (no index on subject_did before) + create index if not exists idx_follows_subject_did on follows(subject_did); + -- global timeline: newest follows without a full sort + create index if not exists idx_follows_followed_at on follows(followed_at); + -- global timeline: newest repos without a full sort + create index if not exists idx_repos_created on repos(created); + `) + return err + }) + return &DB{ db, logger, diff --git a/appview/db/timeline.go b/appview/db/timeline.go --- a/appview/db/timeline.go +++ b/appview/db/timeline.go @@ -8,40 +8,39 @@ "tangled.org/core/orm" ) +// followingFilter compiles to `key in (select subject_did from follows ...)`, +// keeping the following-set check inside sqlite rather than materializing the +// followed dids into a huge placeholder list. +func followingFilter(key, loggedInUserDid string) orm.Filter { + return orm.FilterInSubquery(key, "select subject_did from follows where user_did = ?", loggedInUserDid) +} + // TODO: this gathers heterogenous events from different sources and aggregates // them in code; if we did this entirely in sql, we could order and limit and paginate easily func MakeTimeline(e Execer, limit int, loggedInUserDid string, limitToUsersIsFollowing bool) ([]models.TimelineGroup, error) { var events []models.TimelineEvent - - var userIsFollowing []string - if limitToUsersIsFollowing { - following, err := GetFollowing(e, loggedInUserDid) - if err != nil { - return nil, err - } - - userIsFollowing = make([]string, 0, len(following)) - for _, follow := range following { - userIsFollowing = append(userIsFollowing, follow.SubjectDid) - } - } // Fetch more events than we need to so that when we collapse each individual // event into groups, we can still be relatively confident that we will have // `limit` groups to fill the timeline with. Adjust multiplier as necessary. fetchLimit := limit * 2 - repos, err := getTimelineRepos(e, fetchLimit, loggedInUserDid, userIsFollowing) + var followingOnly string + if limitToUsersIsFollowing { + followingOnly = loggedInUserDid + } + + repos, err := getTimelineRepos(e, fetchLimit, loggedInUserDid, followingOnly) if err != nil { return nil, err } - stars, err := getTimelineStars(e, fetchLimit, loggedInUserDid, userIsFollowing) + stars, err := getTimelineStars(e, fetchLimit, loggedInUserDid, followingOnly) if err != nil { return nil, err } - follows, err := getTimelineFollows(e, fetchLimit, loggedInUserDid, userIsFollowing) + follows, err := getTimelineFollows(e, fetchLimit, loggedInUserDid, followingOnly) if err != nil { return nil, err } @@ -122,10 +121,10 @@ return isStarred, starCount } -func getTimelineRepos(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) { +func getTimelineRepos(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) { filters := make([]orm.Filter, 0) - if userIsFollowing != nil { - filters = append(filters, orm.FilterIn("did", userIsFollowing)) + if followingOnly != "" { + filters = append(filters, followingFilter("did", followingOnly)) } repos, err := GetReposPaginated(e, pagination.Page{Limit: limit}, filters...) @@ -182,10 +181,10 @@ return events, nil } -func getTimelineStars(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) { +func getTimelineStars(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) { filters := make([]orm.Filter, 0) - if userIsFollowing != nil { - filters = append(filters, orm.FilterIn("did", userIsFollowing)) + if followingOnly != "" { + filters = append(filters, followingFilter("did", followingOnly)) } stars, err := GetRepoStars(e, pagination.Page{Limit: limit}, filters...) @@ -218,10 +217,10 @@ return events, nil } -func getTimelineFollows(e Execer, limit int, loggedInUserDid string, userIsFollowing []string) ([]models.TimelineEvent, error) { +func getTimelineFollows(e Execer, limit int, loggedInUserDid string, followingOnly string) ([]models.TimelineEvent, error) { filters := make([]orm.Filter, 0) - if userIsFollowing != nil { - filters = append(filters, orm.FilterIn("user_did", userIsFollowing)) + if followingOnly != "" { + filters = append(filters, followingFilter("user_did", followingOnly)) } follows, err := GetFollows(e, limit, filters...) diff --git a/appview/db/timeline_test.go b/appview/db/timeline_test.go new file mode 100644 --- /dev/null +++ b/appview/db/timeline_test.go @@ -0,0 +1,125 @@ +package db + +import ( + "testing" + + "tangled.org/core/orm" +) + +func seedFollow(t *testing.T, d *DB, userDid, subjectDid, rkey, followedAt string) { + t.Helper() + if _, err := d.Exec( + `insert into follows (user_did, subject_did, rkey, followed_at) values (?, ?, ?, ?)`, + userDid, subjectDid, rkey, followedAt, + ); err != nil { + t.Fatalf("seedFollow %s -> %s: %v", userDid, subjectDid, err) + } +} + +func seedStar(t *testing.T, d *DB, did, rkey, subject, created string) { + t.Helper() + if _, err := d.Exec( + `insert into stars (did, rkey, subject_type, subject, created) values (?, ?, 'repo', ?, ?)`, + did, rkey, subject, created, + ); err != nil { + t.Fatalf("seedStar %s -> %s: %v", did, subject, err) + } +} + +func TestFilterInSubquery(t *testing.T) { + f := orm.FilterInSubquery("did", "select subject_did from follows where user_did = ?", "did:plc:viewer") + if got, want := f.Condition(), "did in (select subject_did from follows where user_did = ?)"; got != want { + t.Errorf("Condition() = %q, want %q", got, want) + } + if args := f.Arg(); len(args) != 1 || args[0] != "did:plc:viewer" { + t.Errorf("Arg() = %v, want [did:plc:viewer]", args) + } +} + +func TestMakeTimeline_FollowingOnly(t *testing.T) { + d := newTestDB(t) + + const ( + viewer = "did:plc:viewer" + followed = "did:plc:followed" + stranger = "did:plc:stranger" + ) + + // viewer follows `followed` but not `stranger` + seedFollow(t, d, viewer, followed, "rkey-viewer-followed", "2024-01-01T00:00:00Z") + + // both users create repos + seedRepo(t, d, followed, "knot.example.com", "followed-repo", "rkey-fr", "did:plc:repo-followed") + seedRepo(t, d, stranger, "knot.example.com", "stranger-repo", "rkey-sr", "did:plc:repo-stranger") + + // both users star a repo + seedStar(t, d, followed, "rkey-fs", "did:plc:repo-stranger", "2024-02-01T00:00:00Z") + seedStar(t, d, stranger, "rkey-ss", "did:plc:repo-followed", "2024-02-01T00:00:00Z") + + // both users follow someone else + seedFollow(t, d, followed, stranger, "rkey-ff", "2024-03-01T00:00:00Z") + seedFollow(t, d, stranger, viewer, "rkey-sf", "2024-03-01T00:00:00Z") + + groups, err := MakeTimeline(d, 50, viewer, true) + if err != nil { + t.Fatalf("MakeTimeline(following): %v", err) + } + + var nRepos, nStars, nFollows int + for _, g := range groups { + switch { + case g.Primary.Repo != nil: + nRepos++ + if g.Primary.Repo.Did != followed { + t.Errorf("repo event from %q, want only %q", g.Primary.Repo.Did, followed) + } + case g.Primary.RepoStar != nil: + nStars++ + if g.Primary.RepoStar.Star.Did != followed { + t.Errorf("star event from %q, want only %q", g.Primary.RepoStar.Star.Did, followed) + } + case g.Primary.Follow != nil: + nFollows++ + if g.Primary.Follow.UserDid != followed { + t.Errorf("follow event from %q, want only %q", g.Primary.Follow.UserDid, followed) + } + } + } + + if nRepos != 1 || nStars != 1 || nFollows != 1 { + t.Errorf("got %d repo, %d star, %d follow events; want 1 of each", nRepos, nStars, nFollows) + } +} + +func TestMakeTimeline_FollowingNobody(t *testing.T) { + d := newTestDB(t) + + // other users are active, but viewer follows nobody + seedRepo(t, d, "did:plc:stranger", "knot.example.com", "repo", "rkey-r", "did:plc:repo-1") + seedStar(t, d, "did:plc:stranger", "rkey-s", "did:plc:repo-1", "2024-02-01T00:00:00Z") + seedFollow(t, d, "did:plc:stranger", "did:plc:other", "rkey-f", "2024-03-01T00:00:00Z") + + groups, err := MakeTimeline(d, 50, "did:plc:viewer", true) + if err != nil { + t.Fatalf("MakeTimeline(following nobody): %v", err) + } + if len(groups) != 0 { + t.Errorf("expected empty following timeline, got %d groups", len(groups)) + } +} + +func TestMakeTimeline_Global(t *testing.T) { + d := newTestDB(t) + + seedRepo(t, d, "did:plc:a", "knot.example.com", "repo-a", "rkey-a", "did:plc:repo-a") + seedStar(t, d, "did:plc:b", "rkey-bs", "did:plc:repo-a", "2024-02-01T00:00:00Z") + seedFollow(t, d, "did:plc:b", "did:plc:a", "rkey-bf", "2024-03-01T00:00:00Z") + + groups, err := MakeTimeline(d, 50, "", false) + if err != nil { + t.Fatalf("MakeTimeline(global): %v", err) + } + if len(groups) != 3 { + t.Errorf("expected 3 groups in global timeline, got %d", len(groups)) + } +} -- tangled.sh