diff --git a/appview/db/collaborators.go b/appview/db/collaborators.go --- a/appview/db/collaborators.go +++ b/appview/db/collaborators.go @@ -59,7 +59,7 @@ return nil, nil } - return GetRepos(e, 0, orm.FilterIn("at_uri", repoAts)) + return GetRepos(e, orm.FilterIn("at_uri", repoAts)) } func GetCollaborators(e Execer, filters ...orm.Filter) ([]models.Collaborator, error) { diff --git a/appview/db/issues.go b/appview/db/issues.go --- a/appview/db/issues.go +++ b/appview/db/issues.go @@ -206,7 +206,7 @@ repoAts = append(repoAts, string(issue.RepoAt)) } - repos, err := GetRepos(e, 0, orm.FilterIn("at_uri", repoAts)) + repos, err := GetRepos(e, orm.FilterIn("at_uri", repoAts)) if err != nil { return nil, fmt.Errorf("failed to build repo mappings: %w", err) } diff --git a/appview/db/profile.go b/appview/db/profile.go --- a/appview/db/profile.go +++ b/appview/db/profile.go @@ -66,7 +66,7 @@ *items = append(*items, &issue) } - repos, err := GetRepos(e, 0, orm.FilterEq("did", forDid)) + repos, err := GetRepos(e, orm.FilterEq("did", forDid)) if err != nil { return nil, fmt.Errorf("error getting all repos by did: %w", err) } @@ -489,7 +489,7 @@ } // ensure all pinned repos are either own repos or collaborating repos - repos, err := GetRepos(e, 0, orm.FilterEq("did", profile.Did)) + repos, err := GetRepos(e, orm.FilterEq("did", profile.Did)) if err != nil { log.Printf("getting repos for %s: %s", profile.Did, err) } diff --git a/appview/db/pulls.go b/appview/db/pulls.go --- a/appview/db/pulls.go +++ b/appview/db/pulls.go @@ -264,7 +264,7 @@ sourceAts = append(sourceAts, *p.PullSource.RepoAt) } } - sourceRepos, err := GetRepos(e, 0, orm.FilterIn("at_uri", sourceAts)) + sourceRepos, err := GetRepos(e, orm.FilterIn("at_uri", sourceAts)) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("failed to get source repos: %w", err) } diff --git a/appview/db/repos.go b/appview/db/repos.go --- a/appview/db/repos.go +++ b/appview/db/repos.go @@ -11,12 +11,15 @@ "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/appview/models" + "tangled.org/core/appview/pagination" "tangled.org/core/orm" ) -func GetRepos(e Execer, limit int, filters ...orm.Filter) ([]models.Repo, error) { - repoMap := make(map[syntax.ATURI]*models.Repo) +func GetRepos(e Execer, filters ...orm.Filter) ([]models.Repo, error) { + return GetReposPaginated(e, pagination.Page{}, filters...) +} +func GetReposPaginated(e Execer, page pagination.Page, filters ...orm.Filter) ([]models.Repo, error) { var conditions []string var args []any for _, filter := range filters { @@ -29,13 +32,14 @@ whereClause = " where " + strings.Join(conditions, " and ") } - limitClause := "" - if limit != 0 { - limitClause = fmt.Sprintf(" limit %d", limit) + pageClause := "" + if page.Limit != 0 { + pageClause = fmt.Sprintf(" limit %d offset %d", page.Limit, page.Offset) } - repoQuery := fmt.Sprintf( - `select + // main query to get repos with pagination + query := fmt.Sprintf(` + select id, did, name, @@ -47,20 +51,19 @@ topics, source, spindle - from - repos r + from repos %s order by created desc - %s`, - whereClause, - limitClause, - ) - rows, err := e.Query(repoQuery, args...) + %s + `, whereClause, pageClause) + + rows, err := e.Query(query, args...) if err != nil { - return nil, fmt.Errorf("failed to execute repo query: %w ", err) + return nil, err } defer rows.Close() + repoMap := make(map[syntax.ATURI]*models.Repo) for rows.Next() { var repo models.Repo var createdAt string @@ -80,12 +83,15 @@ &spindle, ) if err != nil { - return nil, fmt.Errorf("failed to execute repo query: %w ", err) + return nil, err } + // parse created timestamp if t, err := time.Parse(time.RFC3339, createdAt); err == nil { repo.Created = t } + + // handle nullable fields if description.Valid { repo.Description = description.String } @@ -107,66 +113,66 @@ } if err = rows.Err(); err != nil { - return nil, fmt.Errorf("failed to execute repo query: %w ", err) + return nil, err } + // if no repos, return early + if len(repoMap) == 0 { + return nil, nil + } + + // build IN clause for related queries inClause := strings.TrimSuffix(strings.Repeat("?, ", len(repoMap)), ", ") args = make([]any, len(repoMap)) - i := 0 for _, r := range repoMap { args[i] = r.RepoAt() i++ } - // Get labels for all repos + // get labels for all repos labelsQuery := fmt.Sprintf( `select repo_at, label_at from repo_labels where repo_at in (%s)`, inClause, ) + rows, err = e.Query(labelsQuery, args...) if err != nil { - return nil, fmt.Errorf("failed to execute labels query: %w ", err) + return nil, err } defer rows.Close() for rows.Next() { var repoat, labelat string if err := rows.Scan(&repoat, &labelat); err != nil { - log.Println("err", "err", err) continue } if r, ok := repoMap[syntax.ATURI(repoat)]; ok { r.Labels = append(r.Labels, labelat) } } - if err = rows.Err(); err != nil { - return nil, fmt.Errorf("failed to execute labels query: %w ", err) - } - languageQuery := fmt.Sprintf( - ` + // get primary language for all repos + languageQuery := fmt.Sprintf(` select repo_at, language from ( select - repo_at, - language, - row_number() over ( - partition by repo_at - order by bytes desc - ) as rn + repo_at, language, + row_number() over ( + partition by repo_at + order by bytes desc + ) as rn from repo_languages where repo_at in (%s) - and is_default_ref = 1 - and language <> '' + and is_default_ref = 1 + and language <> '' ) where rn = 1 - `, - inClause, - ) + `, inClause) + rows, err = e.Query(languageQuery, args...) if err != nil { - return nil, fmt.Errorf("failed to execute lang query: %w ", err) + return nil, fmt.Errorf("failed to execute lang query: %w", err) } defer rows.Close() @@ -181,20 +187,18 @@ } } if err = rows.Err(); err != nil { - return nil, fmt.Errorf("failed to execute lang query: %w ", err) + return nil, fmt.Errorf("failed to execute lang query: %w", err) } + // get star counts starCountQuery := fmt.Sprintf( - `select - subject_at, count(1) - from stars - where subject_at in (%s) - group by subject_at`, + `select subject_at, count(1) from stars where subject_at in (%s) group by subject_at`, inClause, ) + rows, err = e.Query(starCountQuery, args...) if err != nil { - return nil, fmt.Errorf("failed to execute star-count query: %w ", err) + return nil, fmt.Errorf("failed to execute star-count query: %w", err) } defer rows.Close() @@ -210,22 +214,23 @@ } } if err = rows.Err(); err != nil { - return nil, fmt.Errorf("failed to execute star-count query: %w ", err) + return nil, fmt.Errorf("failed to execute star-count query: %w", err) } - issueCountQuery := fmt.Sprintf( - `select + // get issue counts + issueCountQuery := fmt.Sprintf(` + select repo_at, count(case when open = 1 then 1 end) as open_count, count(case when open = 0 then 1 end) as closed_count from issues where repo_at in (%s) - group by repo_at`, - inClause, - ) + group by repo_at + `, inClause) + rows, err = e.Query(issueCountQuery, args...) if err != nil { - return nil, fmt.Errorf("failed to execute issue-count query: %w ", err) + return nil, fmt.Errorf("failed to execute issue-count query: %w", err) } defer rows.Close() @@ -242,11 +247,12 @@ } } if err = rows.Err(); err != nil { - return nil, fmt.Errorf("failed to execute issue-count query: %w ", err) + return nil, fmt.Errorf("failed to execute issue-count query: %w", err) } - pullCountQuery := fmt.Sprintf( - `select + // get pull counts + pullCountQuery := fmt.Sprintf(` + select repo_at, count(case when state = ? then 1 end) as open_count, count(case when state = ? then 1 end) as merged_count, @@ -254,21 +260,19 @@ count(case when state = ? then 1 end) as deleted_count from pulls where repo_at in (%s) - group by repo_at`, - inClause, - ) - args = append([]any{ + group by repo_at + `, inClause) + + pullArgs := append([]any{ models.PullOpen, models.PullMerged, models.PullClosed, models.PullDeleted, }, args...) - rows, err = e.Query( - pullCountQuery, - args..., - ) + + rows, err = e.Query(pullCountQuery, pullArgs...) if err != nil { - return nil, fmt.Errorf("failed to execute pulls-count query: %w ", err) + return nil, fmt.Errorf("failed to execute pulls-count query: %w", err) } defer rows.Close() @@ -287,7 +291,7 @@ } } if err = rows.Err(); err != nil { - return nil, fmt.Errorf("failed to execute pulls-count query: %w ", err) + return nil, fmt.Errorf("failed to execute pulls-count query: %w", err) } var repos []models.Repo @@ -295,6 +299,7 @@ repos = append(repos, *r) } + // sort by created timestamp (desc) slices.SortFunc(repos, func(a, b models.Repo) int { if a.Created.After(b.Created) { return -1 @@ -307,7 +312,7 @@ // helper to get exactly one repo func GetRepo(e Execer, filters ...orm.Filter) (*models.Repo, error) { - repos, err := GetRepos(e, 0, filters...) + repos, err := GetReposPaginated(e, pagination.Page{Limit: 1}, filters...) if err != nil { return nil, err } @@ -317,7 +322,7 @@ } if len(repos) != 1 { - return nil, fmt.Errorf("too many rows returned") + return nil, fmt.Errorf("too few rows returned") } return &repos[0], nil diff --git a/appview/db/star.go b/appview/db/star.go --- a/appview/db/star.go +++ b/appview/db/star.go @@ -197,7 +197,7 @@ return nil, nil } - repos, err := GetRepos(e, 0, orm.FilterIn("at_uri", args)) + repos, err := GetRepos(e, orm.FilterIn("at_uri", args)) if err != nil { return nil, err } @@ -300,7 +300,7 @@ } // get full repo data - repos, err := GetRepos(e, 0, orm.FilterIn("at_uri", repoUris)) + repos, err := GetRepos(e, orm.FilterIn("at_uri", repoUris)) if err != nil { return nil, err } diff --git a/appview/db/timeline.go b/appview/db/timeline.go --- a/appview/db/timeline.go +++ b/appview/db/timeline.go @@ -5,6 +5,7 @@ "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/appview/models" + "tangled.org/core/appview/pagination" "tangled.org/core/orm" ) @@ -90,7 +91,7 @@ filters = append(filters, orm.FilterIn("did", userIsFollowing)) } - repos, err := GetRepos(e, limit, filters...) + repos, err := GetReposPaginated(e, pagination.Page{Limit: limit}, filters...) if err != nil { return nil, err } @@ -105,7 +106,7 @@ var origRepos []models.Repo if args != nil { - origRepos, err = GetRepos(e, 0, orm.FilterIn("at_uri", args)) + origRepos, err = GetRepos(e, orm.FilterIn("at_uri", args)) } if err != nil { return nil, err diff --git a/appview/knots/knots.go b/appview/knots/knots.go --- a/appview/knots/knots.go +++ b/appview/knots/knots.go @@ -112,7 +112,6 @@ repos, err := db.GetRepos( k.Db, - 0, orm.FilterEq("knot", domain), ) if err != nil { diff --git a/appview/spindles/spindles.go b/appview/spindles/spindles.go --- a/appview/spindles/spindles.go +++ b/appview/spindles/spindles.go @@ -109,7 +109,6 @@ repos, err := db.GetRepos( s.Db, - 0, orm.FilterEq("spindle", instance), ) if err != nil { diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -107,7 +107,6 @@ var errWebhook error repos, err := db.GetRepos( d, - 0, orm.FilterEq("did", record.RepoDid), orm.FilterEq("name", record.RepoName), ) @@ -149,7 +148,6 @@ repos, err := db.GetRepos( d, - 0, orm.FilterEq("did", record.RepoDid), orm.FilterEq("name", record.RepoName), ) @@ -241,7 +239,6 @@ repos, err := db.GetRepos( d, - 0, orm.FilterEq("did", record.RepoDid), orm.FilterEq("name", record.RepoName), ) @@ -307,7 +304,6 @@ // does this repo have a spindle configured? repos, err := db.GetRepos( d, - 0, orm.FilterEq("did", record.TriggerMetadata.Repo.Did), orm.FilterEq("name", record.TriggerMetadata.Repo.Repo), ) diff --git a/appview/state/profile.go b/appview/state/profile.go --- a/appview/state/profile.go +++ b/appview/state/profile.go @@ -139,7 +139,6 @@ repos, err := db.GetRepos( s.db, - 0, orm.FilterEq("did", profile.UserDid), ) if err != nil { @@ -231,7 +230,6 @@ repos, err := db.GetRepos( s.db, - 0, orm.FilterEq("did", profile.UserDid), ) if err != nil { @@ -749,7 +747,7 @@ profile = &models.Profile{Did: user.Active.Did} } - repos, err := db.GetRepos(s.db, 0, orm.FilterEq("did", user.Active.Did)) + repos, err := db.GetRepos(s.db, orm.FilterEq("did", user.Active.Did)) if err != nil { log.Printf("getting repos for %s: %s", user.Active.Did, err) }