From 451de824d8ad65faa13bb36cef1f9243bb4234ed Mon Sep 17 00:00:00 2001 From: Aly Raffauf Date: Fri, 10 Jul 2026 16:21:38 -0400 Subject: [PATCH] pagination: add fetchAllPages helper and use in all list functions --- atproto/records.go | 29 +++++++++++++++++++++++++ internal/cli/ssh_key_list.go | 4 ++-- tangled/list.go | 41 +++++++++++++++++++++++++++++++++--- tangled/list_issues.go | 14 +++++++++--- tangled/list_pulls.go | 14 +++++++++--- tangled/list_repos.go | 20 ++++++++++++------ 6 files changed, 105 insertions(+), 17 deletions(-) diff --git a/atproto/records.go b/atproto/records.go index 0c33a47..dd0f432 100644 --- a/atproto/records.go +++ b/atproto/records.go @@ -45,6 +45,10 @@ type ListRecordsOpts struct { Reverse bool } +// maxRecordPages caps how many pages ListAllRecords will follow, as a +// safety net against a server that never returns an empty cursor. +const maxRecordPages = 1000 + // PutRecord writes a record to the PDS, returning its at:// URI and CID. func (a *ATProto) PutRecord(ctx context.Context, in PutRecordInput) (uri, cid string, err error) { var out struct { @@ -90,3 +94,28 @@ func (a *ATProto) ListRecords(ctx context.Context, repo, collection string, opts } return &out, nil } + +// ListAllRecords fetches every record in collection for repo, following +// pagination cursors until the listing is exhausted. opts.Limit sets the +// page size; opts.Cursor is ignored since pagination always starts from +// the first page. +func (a *ATProto) ListAllRecords(ctx context.Context, repo, collection string, opts ListRecordsOpts) ([]RecordItem, error) { + var all []RecordItem + cursor := "" + + for range maxRecordPages { + opts.Cursor = cursor + out, err := a.ListRecords(ctx, repo, collection, opts) + if err != nil { + return nil, err + } + all = append(all, out.Records...) + + if out.Cursor == nil || *out.Cursor == "" { + return all, nil + } + cursor = *out.Cursor + } + + return nil, fmt.Errorf("exceeded %d pages listing %s records for %q", maxRecordPages, collection, repo) +} diff --git a/internal/cli/ssh_key_list.go b/internal/cli/ssh_key_list.go index 5de9a56..7cbb777 100644 --- a/internal/cli/ssh_key_list.go +++ b/internal/cli/ssh_key_list.go @@ -36,12 +36,12 @@ If no argument is given, lists the authenticated user's keys } atClient := &atproto.ATProto{Client: &atclient.APIClient{Host: pdsURL}} - out, err := atClient.ListRecords(ctx, ident.DID.String(), "sh.tangled.publicKey", atproto.ListRecordsOpts{Limit: defaultListLimit}) + records, err := atClient.ListAllRecords(ctx, ident.DID.String(), "sh.tangled.publicKey", atproto.ListRecordsOpts{Limit: defaultListLimit}) if err != nil { return fmt.Errorf("list SSH keys for %q: %w", handle, err) } - items := buildSSHKeyItems(out.Records) + items := buildSSHKeyItems(records) return output(items, renderSSHKeyList) }, } diff --git a/tangled/list.go b/tangled/list.go index ccb2907..60ca102 100644 --- a/tangled/list.go +++ b/tangled/list.go @@ -1,6 +1,14 @@ package tangled -import "encoding/json" +import ( + "context" + "encoding/json" + "fmt" +) + +// maxPaginationPages caps how many pages fetchAllPages will follow, as a +// safety net against a server that never returns an empty cursor. +const maxPaginationPages = 1000 // ListItem is one item in an issue or pull-request listing. type ListItem struct { @@ -26,8 +34,9 @@ type ListOpts struct { Order string // "asc" or "desc" } -// params builds the XRPC query parameters for subject. -func (o ListOpts) params(subject string) map[string]any { +// params builds the XRPC query parameters for subject, requesting the page +// after cursor (the first page if cursor is empty). +func (o ListOpts) params(subject, cursor string) map[string]any { params := map[string]any{"subject": subject} if o.Author != "" { params["author"] = o.Author @@ -43,5 +52,31 @@ func (o ListOpts) params(subject string) map[string]any { if o.Order != "" { params["order"] = o.Order } + if cursor != "" { + params["cursor"] = cursor + } return params } + +// fetchAllPages calls fetch for successive pages, advancing the cursor it +// returns, until a page reports no further cursor. It returns every item +// across all pages combined. +func fetchAllPages[T any](ctx context.Context, fetch func(ctx context.Context, cursor string) (items []T, nextCursor *string, err error)) ([]T, error) { + var all []T + cursor := "" + + for page := 0; page < maxPaginationPages; page++ { + items, nextCursor, err := fetch(ctx, cursor) + if err != nil { + return nil, err + } + all = append(all, items...) + + if nextCursor == nil || *nextCursor == "" { + return all, nil + } + cursor = *nextCursor + } + + return nil, fmt.Errorf("exceeded %d pages without reaching the end of the list", maxPaginationPages) +} diff --git a/tangled/list_issues.go b/tangled/list_issues.go index b48081f..1626174 100644 --- a/tangled/list_issues.go +++ b/tangled/list_issues.go @@ -17,10 +17,18 @@ type IssueRecord struct { References []string `json:"references,omitempty"` } +// ListIssues fetches every issue for repoDid, following pagination +// cursors until the listing is exhausted. func (t *Tangled) ListIssues(ctx context.Context, repoDid string, opts ListOpts) (*List, error) { - var out List - if err := t.Client.Get(ctx, syntax.NSID("sh.tangled.repo.listIssues"), opts.params(repoDid), &out); err != nil { + items, err := fetchAllPages(ctx, func(ctx context.Context, cursor string) ([]ListItem, *string, error) { + var page List + if err := t.Client.Get(ctx, syntax.NSID("sh.tangled.repo.listIssues"), opts.params(repoDid, cursor), &page); err != nil { + return nil, nil, err + } + return page.Items, page.Cursor, nil + }) + if err != nil { return nil, fmt.Errorf("list issues for %q: %w", repoDid, err) } - return &out, nil + return &List{Items: items}, nil } diff --git a/tangled/list_pulls.go b/tangled/list_pulls.go index 7257f4b..4983b5a 100644 --- a/tangled/list_pulls.go +++ b/tangled/list_pulls.go @@ -44,10 +44,18 @@ type PatchBlob struct { Size int64 `json:"size"` } +// ListPulls fetches every pull request for repoDid, following pagination +// cursors until the listing is exhausted. func (t *Tangled) ListPulls(ctx context.Context, repoDid string, opts ListOpts) (*List, error) { - var out List - if err := t.Client.Get(ctx, syntax.NSID("sh.tangled.repo.listPulls"), opts.params(repoDid), &out); err != nil { + items, err := fetchAllPages(ctx, func(ctx context.Context, cursor string) ([]ListItem, *string, error) { + var page List + if err := t.Client.Get(ctx, syntax.NSID("sh.tangled.repo.listPulls"), opts.params(repoDid, cursor), &page); err != nil { + return nil, nil, err + } + return page.Items, page.Cursor, nil + }) + if err != nil { return nil, fmt.Errorf("list PRs for %q: %w", repoDid, err) } - return &out, nil + return &List{Items: items}, nil } diff --git a/tangled/list_repos.go b/tangled/list_repos.go index c16a766..c8c1fc9 100644 --- a/tangled/list_repos.go +++ b/tangled/list_repos.go @@ -12,15 +12,23 @@ type RepoList struct { Cursor *string `json:"cursor"` } +// ListRepos fetches every repo owned by ownerDid, following pagination +// cursors until the listing is exhausted. func (t *Tangled) ListRepos(ctx context.Context, ownerDid string) (*RepoList, error) { - var repos RepoList - err := t.Client.Get(ctx, syntax.NSID("sh.tangled.repo.listRepos"), map[string]any{ - "subject": ownerDid, - "limit": 100, - }, &repos) + items, err := fetchAllPages(ctx, func(ctx context.Context, cursor string) ([]Repo, *string, error) { + params := map[string]any{"subject": ownerDid, "limit": 100} + if cursor != "" { + params["cursor"] = cursor + } + var page RepoList + if err := t.Client.Get(ctx, syntax.NSID("sh.tangled.repo.listRepos"), params, &page); err != nil { + return nil, nil, err + } + return page.Items, page.Cursor, nil + }) if err != nil { return nil, fmt.Errorf("list tangled repos for %q: %w", ownerDid, err) } - return &repos, nil + return &RepoList{Items: items}, nil } -- 2.51.2