From 2af3617a77380d60c734f6fb6f2b3bb0704a3874 Mon Sep 17 00:00:00 2001 From: Eric Davis Date: Tue, 19 Nov 2024 20:02:13 -0800 Subject: [PATCH] feat: commit/checkpoint every 100k --- cmd/bsky-users/main.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/cmd/bsky-users/main.go b/cmd/bsky-users/main.go index 6f516b8..5db1971 100644 --- a/cmd/bsky-users/main.go +++ b/cmd/bsky-users/main.go @@ -133,26 +133,24 @@ func handler(ctx context.Context, queue *Queue, dbCnx *sql.DB) { dbTx.ExecContext(ctx, userTimestampUpdate, did, ts, ts) eventCount += 1 - if eventCount%2500 == 0 { + if eventCount%100_000 == 0 { if err = dbTx.Commit(); err != nil { log.Printf("commit failed: %v\n", err) } else { log.Printf("commit successful\n") } - if eventCount%25_000 == 0 { - var results CheckpointResults - err = dbCnx.QueryRowContext(ctx, "PRAGMA wal_checkpoint(RESTART)").Scan(&results.Blocked, &results.Pages, &results.Transferred) - switch { - case err != nil: - log.Printf("failed checkpoint: %v\n", err) - case results.Blocked == 1: - log.Printf("checkpoint: blocked\n") - case results.Pages == results.Transferred: - log.Printf("checkpoint: %d pages transferred\n", results.Transferred) - case results.Pages != results.Transferred: - log.Printf("checkpoint: %d pages, %d transferred\n", results.Pages, results.Transferred) - } + var results CheckpointResults + err = dbCnx.QueryRowContext(ctx, "PRAGMA wal_checkpoint(RESTART)").Scan(&results.Blocked, &results.Pages, &results.Transferred) + switch { + case err != nil: + log.Printf("failed checkpoint: %v\n", err) + case results.Blocked == 1: + log.Printf("checkpoint: blocked\n") + case results.Pages == results.Transferred: + log.Printf("checkpoint: %d pages transferred\n", results.Transferred) + case results.Pages != results.Transferred: + log.Printf("checkpoint: %d pages, %d transferred\n", results.Pages, results.Transferred) } dbTx, err = dbCnx.BeginTx(ctx, nil) -- 2.51.2