diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go new file mode 100644 index 00000000..44983cdd --- /dev/null +++ b/cmd/netsync/main.go @@ -0,0 +1,441 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "os/signal" + "sync" + "syscall" + "time" + + logging "github.com/ipfs/go-log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/time/rate" + + "github.com/bluesky-social/indigo/util/version" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("netsync") + +func main() { + app := cli.App{ + Name: "netsync", + Usage: "atproto network cloning tool", + Version: version.Version, + } + + app.Flags = []cli.Flag{ + &cli.IntFlag{ + Name: "port", + Usage: "listen port for metrics server", + Value: 8753, + }, + &cli.IntFlag{ + Name: "worker-count", + Usage: "number of workers to run concurrently", + Value: 10, + }, + &cli.Float64Flag{ + Name: "checkout-limit", + Usage: "maximum number of repos per second to checkout", + Value: 4, + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "directory to write cloned repos to", + Value: "netsync-out", + }, + &cli.StringFlag{ + Name: "repo-list", + Usage: "path to file containing list of repos to clone", + Value: "repos.txt", + }, + &cli.StringFlag{ + Name: "state-file", + Usage: "path to file to write state to", + Value: "state.json", + }, + &cli.StringFlag{ + Name: "checkout-path", + Usage: "path to checkout endpoint", + Value: "https://bsky.social/xrpc/com.atproto.sync.getCheckout", + }, + } + + app.Action = Netsync + + err := app.Run(os.Args) + if err != nil { + log.Fatal(err) + } +} + +type RepoState struct { + Repo string + State string + FinishedAt time.Time +} + +type NetsyncState struct { + EnqueuedRepos map[string]*RepoState + FinishedRepos map[string]*RepoState + StatePath string + OutDir string + CheckoutPath string + + lk sync.RWMutex + wg sync.WaitGroup + exit chan struct{} + limiter *rate.Limiter + workerCount int + client *http.Client +} + +type instrumentedReader struct { + source io.ReadCloser + counter prometheus.Counter +} + +func (r instrumentedReader) Read(b []byte) (int, error) { + n, err := r.source.Read(b) + r.counter.Add(float64(n)) + return n, err +} + +func (r instrumentedReader) Close() error { + var buf [32]byte + var n int + var err error + for err == nil { + n, err = r.source.Read(buf[:]) + r.counter.Add(float64(n)) + } + closeerr := r.source.Close() + if err != nil && err != io.EOF { + return err + } + return closeerr +} + +func (s *NetsyncState) Save() error { + s.lk.RLock() + defer s.lk.RUnlock() + + stateFile, err := os.OpenFile(s.StatePath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return err + } + defer stateFile.Close() + + stateBytes, err := json.Marshal(s) + if err != nil { + return err + } + + _, err = stateFile.Write(stateBytes) + return err +} + +func (s *NetsyncState) Resume() error { + stateFile, err := os.Open(s.StatePath) + if err != nil { + return err + } + + stateBytes, err := io.ReadAll(stateFile) + if err != nil { + return err + } + + err = json.Unmarshal(stateBytes, s) + if err != nil { + return err + } + + return nil +} + +func (s *NetsyncState) Dequeue() string { + s.lk.Lock() + defer s.lk.Unlock() + + for repo, state := range s.EnqueuedRepos { + if state.State == "enqueued" { + state.State = "dequeued" + return repo + } + } + + return "" +} + +func (s *NetsyncState) Finish(repo string, state string) { + s.lk.Lock() + defer s.lk.Unlock() + + s.FinishedRepos[repo] = &RepoState{ + Repo: repo, + State: state, + FinishedAt: time.Now(), + } + + delete(s.EnqueuedRepos, repo) +} + +func Netsync(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Try to resume from state file + state := &NetsyncState{ + StatePath: cctx.String("state-file"), + CheckoutPath: cctx.String("checkout-path"), + OutDir: cctx.String("out-dir"), + workerCount: cctx.Int("worker-count"), + limiter: rate.NewLimiter(rate.Limit(cctx.Float64("checkout-limit")), 1), + exit: make(chan struct{}), + wg: sync.WaitGroup{}, + client: &http.Client{ + Timeout: 180 * time.Second, + }, + } + + // Create out dir + err := os.MkdirAll(state.OutDir, 0755) + if err != nil { + return err + } + + err = state.Resume() + if state.EnqueuedRepos == nil { + state.EnqueuedRepos = make(map[string]*RepoState) + } + + if state.FinishedRepos == nil { + state.FinishedRepos = make(map[string]*RepoState) + } + + if err != nil { + // Read repo list + repoListFile, err := os.Open(cctx.String("repo-list")) + if err != nil { + return err + } + + fileScanner := bufio.NewScanner(repoListFile) + fileScanner.Split(bufio.ScanLines) + + for fileScanner.Scan() { + repo := fileScanner.Text() + state.EnqueuedRepos[repo] = &RepoState{ + Repo: repo, + State: "enqueued", + } + } + } else { + log.Info("Resuming from state file") + } + + // Start metrics server + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + metricsServer := &http.Server{ + Addr: fmt.Sprintf(":%d", cctx.Int("port")), + Handler: mux, + } + + go func() { + state.wg.Add(1) + defer state.wg.Done() + if err := metricsServer.ListenAndServe(); err != http.ErrServerClosed { + log.Fatalf("failed to start metrics server: %+v", err) + } + log.Info("metrics server shut down successfully") + }() + + // Start workers + for i := 0; i < state.workerCount; i++ { + state.wg.Add(1) + go func(id int) { + defer state.wg.Done() + err := state.worker(id) + if err != nil { + log.Errorw("worker failed", "err", err) + } + }(i) + } + + // Check for empty queue + go func() { + state.wg.Add(1) + defer state.wg.Done() + t := time.NewTicker(30 * time.Second) + for { + select { + case <-ctx.Done(): + err := state.Save() + if err != nil { + log.Errorw("failed to save state", "err", err) + } + return + case <-t.C: + err := state.Save() + if err != nil { + log.Errorw("failed to save state", "err", err) + } + state.lk.RLock() + if len(state.EnqueuedRepos) == 0 { + log.Info("no more repos to clone, shutting down") + close(state.exit) + return + } + state.lk.RUnlock() + } + } + }() + + // Trap SIGINT to trigger a shutdown. + log.Info("listening for signals") + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + select { + case <-signals: + cancel() + close(state.exit) + log.Info("shutting down on signal") + case <-ctx.Done(): + cancel() + close(state.exit) + log.Info("shutting down on context done") + case <-state.exit: + cancel() + log.Info("shutting down on exit signal") + } + + log.Info("shutting down, waiting for workers to clean up...") + + if err := metricsServer.Shutdown(ctx); err != nil { + log.Errorf("failed to shut down metrics server: %+v", err) + } + + state.wg.Wait() + + log.Info("shut down successfully") + + return nil + +} + +func (s *NetsyncState) worker(id int) error { + log := log.With("worker", id) + log.Infow("starting worker") + defer log.Infow("worker stopped") + for { + select { + case <-s.exit: + log.Info("worker exiting due to exit signal") + return nil + default: + ctx := context.Background() + // Dequeue repo + repo := s.Dequeue() + if repo == "" { + // No more repos to clone + return nil + } + + // Wait for rate limiter + s.limiter.Wait(ctx) + + // Clone repo + cloneState, err := s.cloneRepo(ctx, repo) + if err != nil { + log.Errorw("failed to clone repo", "repo", repo, "err", err) + } + + // Update state + s.Finish(repo, cloneState) + log.Infow("worker finished", "repo", repo, "status", cloneState) + } + } +} + +var repoCloneDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "netsync_repo_clone_duration_seconds", + Help: "Duration of repo clone operations", +}, []string{"status"}) + +var bytesProcessed = promauto.NewCounter(prometheus.CounterOpts{ + Name: "netsync_bytes_processed", + Help: "Number of bytes processed", +}) + +func (s *NetsyncState) cloneRepo(ctx context.Context, repo string) (cloneState string, err error) { + log := log.With("repo", repo, "source", "cloneRepo") + log.Infow("cloning repo") + + start := time.Now() + defer func() { + duration := time.Since(start) + repoCloneDuration.WithLabelValues(cloneState).Observe(duration.Seconds()) + }() + + var url = fmt.Sprintf("%s?did=%s", s.CheckoutPath, repo) + + // Clone repo + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + cloneState = "failed (request-creation)" + return cloneState, fmt.Errorf("failed to create request: %w", err) + } + + resp, err := s.client.Do(req) + if err != nil { + cloneState = "failed (client.do)" + return cloneState, fmt.Errorf("failed to get repo: %w", err) + } + + if resp.StatusCode != http.StatusOK { + cloneState = fmt.Sprintf("failed (status: %d)", resp.StatusCode) + return cloneState, fmt.Errorf("failed to get repo: %s", resp.Status) + } + + instrumentedReader := instrumentedReader{ + source: resp.Body, + counter: bytesProcessed, + } + defer instrumentedReader.Close() + + // Write to file + outPath := fmt.Sprintf("%s/%s", s.OutDir, repo) + outFile, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + cloneState = "failed (file.open)" + return cloneState, fmt.Errorf("failed to open file: %w", err) + } + + _, err = io.Copy(outFile, instrumentedReader) + if err != nil { + cloneState = "failed (file.copy)" + return cloneState, fmt.Errorf("failed to copy file: %w", err) + } + + err = outFile.Close() + if err != nil { + cloneState = "failed (file.close)" + return cloneState, fmt.Errorf("failed to close file: %w", err) + } + cloneState = "success" + return cloneState, nil +} -- 2.51.2 From 0892046c45036f1568317165110d2748106adcef Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 25 Aug 2023 19:38:23 +0000 Subject: [PATCH 002/192] Track enqueued and finished jobs via prometheus --- .gitignore | 2 ++ cmd/netsync/main.go | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.gitignore b/.gitignore index d3e24cb4..1f2ce849 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ bsky.auth # Sonar cursor file sonar_cursor.json out/ +state.json +netsync-out/ diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 44983cdd..0659989f 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -163,10 +163,17 @@ func (s *NetsyncState) Resume() error { return nil } +var enqueuedJobs = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "netsync_enqueued_jobs", + Help: "Number of enqueued jobs", +}) + func (s *NetsyncState) Dequeue() string { s.lk.Lock() defer s.lk.Unlock() + enqueuedJobs.Set(float64(len(s.EnqueuedRepos))) + for repo, state := range s.EnqueuedRepos { if state.State == "enqueued" { state.State = "dequeued" @@ -177,6 +184,11 @@ func (s *NetsyncState) Dequeue() string { return "" } +var finishedJobs = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "netsync_finished_jobs", + Help: "Number of finished jobs", +}) + func (s *NetsyncState) Finish(repo string, state string) { s.lk.Lock() defer s.lk.Unlock() @@ -187,6 +199,8 @@ func (s *NetsyncState) Finish(repo string, state string) { FinishedAt: time.Now(), } + finishedJobs.Set(float64(len(s.FinishedRepos))) + delete(s.EnqueuedRepos, repo) } @@ -224,6 +238,8 @@ func Netsync(cctx *cli.Context) error { state.FinishedRepos = make(map[string]*RepoState) } + state.OutDir = cctx.String("out-dir") + if err != nil { // Read repo list repoListFile, err := os.Open(cctx.String("repo-list")) -- 2.51.2 From 9e5cce39a18ac7e81da586a9f70df5fdce59d689 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 25 Aug 2023 22:38:33 +0000 Subject: [PATCH 003/192] Support magic headers --- cmd/netsync/main.go | 55 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 0659989f..6d0f7a37 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -14,9 +14,11 @@ import ( "time" logging "github.com/ipfs/go-log" + _ "github.com/joho/godotenv/autoload" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/prometheus/client_golang/prometheus/promhttp" + "golang.org/x/time/rate" "github.com/bluesky-social/indigo/util/version" @@ -68,6 +70,18 @@ func main() { Usage: "path to checkout endpoint", Value: "https://bsky.social/xrpc/com.atproto.sync.getCheckout", }, + &cli.StringFlag{ + Name: "magic-header-key", + Usage: "header key to send with checkout request", + Value: "", + EnvVars: []string{"MAGIC_HEADER_KEY"}, + }, + &cli.StringFlag{ + Name: "magic-header-val", + Usage: "header value to send with checkout request", + Value: "", + EnvVars: []string{"MAGIC_HEADER_VAL"}, + }, } app.Action = Netsync @@ -88,9 +102,12 @@ type NetsyncState struct { EnqueuedRepos map[string]*RepoState FinishedRepos map[string]*RepoState StatePath string - OutDir string CheckoutPath string + outDir string + magicHeaderKey string + magicHeaderVal string + lk sync.RWMutex wg sync.WaitGroup exit chan struct{} @@ -209,26 +226,34 @@ func Netsync(cctx *cli.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - // Try to resume from state file state := &NetsyncState{ StatePath: cctx.String("state-file"), CheckoutPath: cctx.String("checkout-path"), - OutDir: cctx.String("out-dir"), - workerCount: cctx.Int("worker-count"), - limiter: rate.NewLimiter(rate.Limit(cctx.Float64("checkout-limit")), 1), - exit: make(chan struct{}), - wg: sync.WaitGroup{}, + + outDir: cctx.String("out-dir"), + workerCount: cctx.Int("worker-count"), + limiter: rate.NewLimiter(rate.Limit(cctx.Float64("checkout-limit")), 1), + magicHeaderKey: cctx.String("magic-header-key"), + magicHeaderVal: cctx.String("magic-header-val"), + + exit: make(chan struct{}), + wg: sync.WaitGroup{}, client: &http.Client{ Timeout: 180 * time.Second, }, } + if state.magicHeaderKey != "" && state.magicHeaderVal != "" { + log.Info("using magic header") + } + // Create out dir - err := os.MkdirAll(state.OutDir, 0755) + err := os.MkdirAll(state.outDir, 0755) if err != nil { return err } + // Try to resume from state file err = state.Resume() if state.EnqueuedRepos == nil { state.EnqueuedRepos = make(map[string]*RepoState) @@ -238,8 +263,6 @@ func Netsync(cctx *cli.Context) error { state.FinishedRepos = make(map[string]*RepoState) } - state.OutDir = cctx.String("out-dir") - if err != nil { // Read repo list repoListFile, err := os.Open(cctx.String("repo-list")) @@ -326,10 +349,10 @@ func Netsync(cctx *cli.Context) error { signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) select { - case <-signals: + case sig := <-signals: cancel() close(state.exit) - log.Info("shutting down on signal") + log.Infof("shutting down on signal: %+v", sig) case <-ctx.Done(): cancel() close(state.exit) @@ -416,6 +439,12 @@ func (s *NetsyncState) cloneRepo(ctx context.Context, repo string) (cloneState s return cloneState, fmt.Errorf("failed to create request: %w", err) } + req.Header.Set("Accept", "application/vnd.ipld.car") + req.Header.Set("User-Agent", "jaz-atproto-netsync/0.0.1") + if s.magicHeaderKey != "" && s.magicHeaderVal != "" { + req.Header.Set(s.magicHeaderKey, s.magicHeaderVal) + } + resp, err := s.client.Do(req) if err != nil { cloneState = "failed (client.do)" @@ -434,7 +463,7 @@ func (s *NetsyncState) cloneRepo(ctx context.Context, repo string) (cloneState s defer instrumentedReader.Close() // Write to file - outPath := fmt.Sprintf("%s/%s", s.OutDir, repo) + outPath := fmt.Sprintf("%s/%s", s.outDir, repo) outFile, err := os.OpenFile(outPath, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { cloneState = "failed (file.open)" -- 2.51.2 From 828942769e665c94fa0c9c84d25ffd19859f286f Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Sat, 26 Aug 2023 01:26:39 +0000 Subject: [PATCH 004/192] Requeue dequeued jobs on restart, add a command to re-enqueue failed jobs --- cmd/netsync/dash.json | 666 ++++++++++++++++++++++++++++++++++++++++++ cmd/netsync/main.go | 39 +++ 2 files changed, 705 insertions(+) create mode 100644 cmd/netsync/dash.json diff --git a/cmd/netsync/dash.json b/cmd/netsync/dash.json new file mode 100644 index 00000000..7718fa5b --- /dev/null +++ b/cmd/netsync/dash.json @@ -0,0 +1,666 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "series", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "locale" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Jobs Finished" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netsync_enqueued_jobs{}", + "legendFormat": "Jobs Enqueued", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netsync_finished_jobs{}", + "hide": false, + "legendFormat": "Jobs Finished", + "range": true, + "refId": "B" + } + ], + "title": "Job Backlog", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdhms" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(netsync_enqueued_jobs / rate(netsync_finished_jobs{}[$__rate_interval]))", + "instant": false, + "legendFormat": "Seconds Until Completion", + "range": true, + "refId": "A" + } + ], + "title": "Time until Completion", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "netsync_bytes_processed{}", + "legendFormat": "Bytes Read", + "range": true, + "refId": "A" + } + ], + "title": "Backfilled Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "cps" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(netsync_repo_clone_duration_seconds_count{status=\"success\"}[$__rate_interval])", + "legendFormat": "{{status}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(netsync_repo_clone_duration_seconds_count{status!=\"success\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{status}}", + "range": true, + "refId": "B" + } + ], + "title": "Backfilled Repos", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 9 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(process_cpu_seconds_total{job=\"netsync\"}[$__rate_interval])", + "legendFormat": "CPU Usage per Second", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 9 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "process_resident_memory_bytes{job=\"netsync\"}", + "legendFormat": "Bytes Used", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage", + "type": "timeseries" + } + ], + "refresh": "10s", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Netsync", + "uid": "n5sI_QRSz", + "version": 13, + "weekStart": "" +} diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 6d0f7a37..03ce9b48 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -9,6 +9,7 @@ import ( "net/http" "os" "os/signal" + "strings" "sync" "syscall" "time" @@ -84,6 +85,37 @@ func main() { }, } + app.Commands = []*cli.Command{ + { + Name: "retry", + Usage: "requeue failed repos", + Action: func(cctx *cli.Context) error { + state := &NetsyncState{ + StatePath: cctx.String("state-file"), + } + + err := state.Resume() + if err != nil { + return err + } + + // Look through finished repos for failed ones + for _, repoState := range state.FinishedRepos { + // Don't retry repos that failed due to a 400 (they've been deleted) + if strings.HasPrefix(repoState.State, "failed") && repoState.State != "failed (status: 400)" { + state.EnqueuedRepos[repoState.Repo] = &RepoState{ + Repo: repoState.Repo, + State: "enqueued", + } + } + } + + // Save state + return state.Save() + }, + }, + } + app.Action = Netsync err := app.Run(os.Args) @@ -257,6 +289,13 @@ func Netsync(cctx *cli.Context) error { err = state.Resume() if state.EnqueuedRepos == nil { state.EnqueuedRepos = make(map[string]*RepoState) + } else { + // Reset any dequeued repos + for _, repoState := range state.EnqueuedRepos { + if repoState.State == "dequeued" { + repoState.State = "enqueued" + } + } } if state.FinishedRepos == nil { -- 2.51.2 From 561d332c92347275c049295e11cf3f8b2207dae9 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Sat, 26 Aug 2023 05:36:51 +0000 Subject: [PATCH 005/192] Example playback command --- cmd/netsync/main.go | 5 + cmd/netsync/playback.go | 263 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+) create mode 100644 cmd/netsync/playback.go diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 03ce9b48..1ad49b4f 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -114,6 +114,11 @@ func main() { return state.Save() }, }, + { + Name: "playback", + Usage: "playback the contents of a netsync output directory", + Action: Playback, + }, } app.Action = Netsync diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go new file mode 100644 index 00000000..cbf836d8 --- /dev/null +++ b/cmd/netsync/playback.go @@ -0,0 +1,263 @@ +package main + +import ( + "context" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/repo" + "github.com/ipfs/go-cid" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/urfave/cli/v2" +) + +type PlaybackState struct { + EnqueuedRepos map[string]*RepoState + FinishedRepos map[string]*RepoState + + outDir string + + lk sync.RWMutex + wg sync.WaitGroup + exit chan struct{} + workerCount int + + textLen atomic.Uint64 +} + +func (s *PlaybackState) Dequeue() string { + s.lk.Lock() + defer s.lk.Unlock() + + enqueuedJobs.Set(float64(len(s.EnqueuedRepos))) + + for repo, state := range s.EnqueuedRepos { + if state.State == "enqueued" { + state.State = "dequeued" + return repo + } + } + + return "" +} + +func (s *PlaybackState) Finish(repo string, state string) { + s.lk.Lock() + defer s.lk.Unlock() + + s.FinishedRepos[repo] = &RepoState{ + Repo: repo, + State: state, + FinishedAt: time.Now(), + } + + finishedJobs.Set(float64(len(s.FinishedRepos))) + + delete(s.EnqueuedRepos, repo) +} + +func Playback(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + start := time.Now() + + state := &PlaybackState{ + outDir: cctx.String("out-dir"), + workerCount: cctx.Int("worker-count"), + wg: sync.WaitGroup{}, + } + + state.EnqueuedRepos = make(map[string]*RepoState) + state.FinishedRepos = make(map[string]*RepoState) + + state.exit = make(chan struct{}) + + // Start metrics server + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + + metricsServer := &http.Server{ + Addr: fmt.Sprintf(":%d", cctx.Int("port")), + Handler: mux, + } + + go func() { + state.wg.Add(1) + defer state.wg.Done() + if err := metricsServer.ListenAndServe(); err != http.ErrServerClosed { + log.Fatalf("failed to start metrics server: %+v", err) + } + log.Info("metrics server shut down successfully") + }() + + // Load all the repos from the out dir + err := filepath.Walk(state.outDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("failed to walk path: %w", err) + } + + if info.IsDir() { + return nil + } + + state.EnqueuedRepos[info.Name()] = &RepoState{ + Repo: info.Name(), + State: "enqueued", + } + + enqueuedJobs.Inc() + + return nil + }) + if err != nil { + return fmt.Errorf("failed to walk out dir: %w", err) + } + + // Start workers + for i := 0; i < state.workerCount; i++ { + go state.worker(i) + } + + // Check for empty queue + go func() { + state.wg.Add(1) + defer state.wg.Done() + t := time.NewTicker(30 * time.Second) + for { + select { + case <-ctx.Done(): + return + case <-t.C: + state.lk.RLock() + if len(state.EnqueuedRepos) == 0 { + log.Info("no more repos to process, shutting down") + close(state.exit) + return + } + state.lk.RUnlock() + } + } + }() + + // Trap SIGINT to trigger a shutdown. + log.Info("listening for signals") + signals := make(chan os.Signal, 1) + signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) + + select { + case sig := <-signals: + cancel() + close(state.exit) + log.Infof("shutting down on signal: %+v", sig) + case <-ctx.Done(): + cancel() + close(state.exit) + log.Info("shutting down on context done") + case <-state.exit: + cancel() + log.Info("shutting down on exit signal") + } + + log.Info("shutting down, waiting for workers to clean up...") + + if err := metricsServer.Shutdown(ctx); err != nil { + log.Errorf("failed to shut down metrics server: %+v", err) + } + + state.wg.Wait() + + // Print stats + log.Infof("processed %d repos and %d UTF-8 text characters in %s", len(state.FinishedRepos), state.textLen.Load(), time.Since(start)) + log.Info("shut down successfully") + + return nil +} + +func (s *PlaybackState) worker(id int) { + log := log.With("worker", id) + s.wg.Add(1) + defer s.wg.Done() + + for { + select { + case <-s.exit: + return + default: + } + + repo := s.Dequeue() + if repo == "" { + return + } + + processState, err := s.processRepo(context.Background(), repo) + if err != nil { + log.Errorf("failed to process repo (%s): %v", repo, err) + } + + s.Finish(repo, processState) + } +} + +func (s *PlaybackState) processRepo(ctx context.Context, did string) (processState string, err error) { + log := log.With("repo", did) + + log.Debug("processing repo") + + // Open the repo file from the out dir + f, err := os.Open(filepath.Join(s.outDir, did)) + if err != nil { + return "", fmt.Errorf("failed to open repo file: %w", err) + } + defer f.Close() + + r, err := repo.ReadRepoFromCar(ctx, f) + if err != nil { + return "", fmt.Errorf("failed to read repo from car: %w", err) + } + + r.ForEach(ctx, "", func(path string, nodeCid cid.Cid) error { + recordCid, rec, err := r.GetRecord(ctx, path) + if err != nil { + return fmt.Errorf("failed to get record: %w", err) + } + + // Verify that the record cid matches the cid in the event + if recordCid != nodeCid { + log.Errorf("mismatch in record and op cid: %s != %s", recordCid, nodeCid) + return nil + } + + switch rec := rec.(type) { + case *bsky.FeedPost: + log.Debugf("processing feed post: %s", rec.Text) + s.textLen.Add(uint64(len(rec.Text))) + case *bsky.FeedLike: + log.Debugf("processing feed like: %s", rec.Subject.Uri) + case *bsky.FeedRepost: + log.Debugf("processing feed repost: %s", rec.Subject.Uri) + case *bsky.GraphFollow: + log.Debugf("processing graph follow: %s", rec.Subject) + case *bsky.GraphBlock: + log.Debugf("processing graph block: %s", rec.Subject) + case *bsky.ActorProfile: + if rec.DisplayName != nil { + log.Debugf("processing actor profile: %s", *rec.DisplayName) + } + } + + return nil + }) + + return "finished", nil +} -- 2.51.2 From 3fb35debbdecc90cd2b3e858b4df136c3210c140 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Sat, 26 Aug 2023 06:08:33 +0000 Subject: [PATCH 006/192] Add netsync helper make targets --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index e6a3d383..05a2b1fb 100644 --- a/Makefile +++ b/Makefile @@ -115,3 +115,11 @@ sc-fire: # Fires supercollider --hostname alpha.supercollider.jazco.io \ --key-file out/alpha.pem \ --input-file out/alpha_in.cbor + +.PHONY: run-netsync +run-netsync: .env ## Runs netsync for local dev + go run ./cmd/netsync--checkout-limit 40 --worker-count 60 --out-dir ../netsync-out + +.PHONY: netsync-playback +netsync-playback: .env ## Runs netsync for local dev + go run ./cmd/netsync --worker-count 60 --out-dir ../netsync-out_2023_08_25 playback -- 2.51.2 From 32cf57707430e7e125850ed808170789b2bae875 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Sat, 26 Aug 2023 06:12:06 +0000 Subject: [PATCH 007/192] Use a printer to format output" --- cmd/netsync/playback.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index cbf836d8..f723a29a 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -17,6 +17,8 @@ import ( "github.com/ipfs/go-cid" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/urfave/cli/v2" + "golang.org/x/text/language" + "golang.org/x/text/message" ) type PlaybackState struct { @@ -176,8 +178,11 @@ func Playback(cctx *cli.Context) error { state.wg.Wait() + p := message.NewPrinter(language.English) + // Print stats - log.Infof("processed %d repos and %d UTF-8 text characters in %s", len(state.FinishedRepos), state.textLen.Load(), time.Since(start)) + log.Info(p.Sprintf("processed %d repos and %d UTF-8 text characters in %s", + len(state.FinishedRepos), state.textLen.Load(), time.Since(start))) log.Info("shut down successfully") return nil -- 2.51.2 From fbe1bd037e3108e420aaf0bc4c066c5e1aed0168 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 29 Aug 2023 06:21:36 +0000 Subject: [PATCH 008/192] Playback messing with Scylla a bit --- Makefile | 13 ++++++- cmd/bigsky/docker-compose.yml | 13 +++++++ cmd/netsync/main.go | 7 ++++ cmd/netsync/playback.go | 68 ++++++++++++++++++++++++++++++++++- go.mod | 8 ++++- go.sum | 17 +++++++++ 6 files changed, 123 insertions(+), 3 deletions(-) create mode 100644 cmd/bigsky/docker-compose.yml diff --git a/Makefile b/Makefile index 05a2b1fb..8ede7724 100644 --- a/Makefile +++ b/Makefile @@ -122,4 +122,15 @@ run-netsync: .env ## Runs netsync for local dev .PHONY: netsync-playback netsync-playback: .env ## Runs netsync for local dev - go run ./cmd/netsync --worker-count 60 --out-dir ../netsync-out_2023_08_25 playback + go run ./cmd/netsync --worker-count 60 --out-dir ../netsync-out_2023_08_25 playback --scylla-nodes 127.0.0.1:9042 + + +SCYLLA_VERSION := latest +SCYLLA_CPU := 0 + +.PHONY: run-scylla +run-scylla: + @echo "==> Running test instance of Scylla $(SCYLLA_VERSION)" + @docker pull scylladb/scylla:$(SCYLLA_VERSION) + @docker run --name scylla -p 9042:9042 --cpuset-cpus=$(SCYLLA_CPU) --memory 1G --rm -d scylladb/scylla:$(SCYLLA_VERSION) + @until docker exec scylla cqlsh -e "DESCRIBE SCHEMA"; do sleep 2; done diff --git a/cmd/bigsky/docker-compose.yml b/cmd/bigsky/docker-compose.yml new file mode 100644 index 00000000..49946c8a --- /dev/null +++ b/cmd/bigsky/docker-compose.yml @@ -0,0 +1,13 @@ +version: "3.8" +services: + postgres: + image: "postgres:14" + ports: + - "5432:5432" + volumes: + - type: bind + source: /mnt/postgres + target: /var/lib/postgresql/data + restart: always + environment: + POSTGRES_PASSWORD: 33pAstcHDMszLedQah2EVYNgnxbCP diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 1ad49b4f..56261500 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -118,6 +118,13 @@ func main() { Name: "playback", Usage: "playback the contents of a netsync output directory", Action: Playback, + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "scylla-nodes", + Usage: "list of scylla nodes to connect to", + EnvVars: []string{"SCYLLA_NODES"}, + }, + }, }, } diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index f723a29a..a87768da 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -7,15 +7,21 @@ import ( "os" "os/signal" "path/filepath" + "strings" "sync" "sync/atomic" "syscall" "time" + "github.com/araddon/dateparse" "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/repo" + "github.com/gocql/gocql" "github.com/ipfs/go-cid" "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/scylladb/gocqlx/v2" + "github.com/scylladb/gocqlx/v2/table" + "github.com/urfave/cli/v2" "golang.org/x/text/language" "golang.org/x/text/message" @@ -33,6 +39,8 @@ type PlaybackState struct { workerCount int textLen atomic.Uint64 + + ses gocqlx.Session } func (s *PlaybackState) Dequeue() string { @@ -66,6 +74,33 @@ func (s *PlaybackState) Finish(repo string, state string) { delete(s.EnqueuedRepos, repo) } +var postMetadata = table.Metadata{ + Name: "netsync.post", + Columns: []string{"did", "rkey", "content", "created_at"}, + PartKey: []string{"did"}, + SortKey: []string{"rkey"}, +} +var postTable = table.New(postMetadata) + +type Post struct { + Did string + Rkey string + Content string + CreatedAt time.Time +} + +func (s *PlaybackState) SetupSchema() error { + if err := s.ses.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS netsync WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };`); err != nil { + return fmt.Errorf("failed to create keyspace: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.post (did text, rkey text, content text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + return fmt.Errorf("failed to create post table: %w", err) + } + + return nil +} + func Playback(cctx *cli.Context) error { ctx := cctx.Context ctx, cancel := context.WithCancel(ctx) @@ -73,10 +108,22 @@ func Playback(cctx *cli.Context) error { start := time.Now() + cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) + session, err := gocqlx.WrapSession(cluster.CreateSession()) + if err != nil { + return fmt.Errorf("failed to create scylla session: %w", err) + } + state := &PlaybackState{ outDir: cctx.String("out-dir"), workerCount: cctx.Int("worker-count"), wg: sync.WaitGroup{}, + ses: session, + } + + err = state.SetupSchema() + if err != nil { + return fmt.Errorf("failed to setup schema: %w", err) } state.EnqueuedRepos = make(map[string]*RepoState) @@ -103,7 +150,7 @@ func Playback(cctx *cli.Context) error { }() // Load all the repos from the out dir - err := filepath.Walk(state.outDir, func(path string, info os.FileInfo, err error) error { + err = filepath.Walk(state.outDir, func(path string, info os.FileInfo, err error) error { if err != nil { return fmt.Errorf("failed to walk path: %w", err) } @@ -243,10 +290,29 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } + rkey := strings.Split(path, "/")[1] + switch rec := rec.(type) { case *bsky.FeedPost: log.Debugf("processing feed post: %s", rec.Text) s.textLen.Add(uint64(len(rec.Text))) + recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) + if err != nil { + log.Errorf("failed to parse created at: %w", err) + return nil + } + + insertPost := postTable.InsertQuery(s.ses) + insertPost.BindStruct(Post{ + Did: did, + Rkey: rkey, + Content: rec.Text, + CreatedAt: recCreatedAt, + }) + if err := insertPost.ExecRelease(); err != nil { + log.Errorf("failed to insert post: %+v", err) + return nil + } case *bsky.FeedLike: log.Debugf("processing feed like: %s", rec.Subject.Uri) case *bsky.FeedRepost: diff --git a/go.mod b/go.mod index 0e0804ed..64ee553f 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/brianvoe/gofakeit/v6 v6.20.2 github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 github.com/goccy/go-json v0.10.2 + github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/gorilla/websocket v1.5.0 github.com/hashicorp/go-retryablehttp v0.7.2 @@ -40,6 +41,7 @@ require ( github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 + github.com/scylladb/gocqlx/v2 v2.8.0 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.1 github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 @@ -57,6 +59,7 @@ require ( golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/sync v0.1.0 + golang.org/x/text v0.12.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.8.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 @@ -83,8 +86,10 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-blockservice v0.5.0 // indirect @@ -126,6 +131,7 @@ require ( github.com/prometheus/procfs v0.9.0 // indirect github.com/prometheus/statsd_exporter v0.22.7 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/scylladb/go-reflectx v1.0.1 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -141,12 +147,12 @@ require ( golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/sys v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect google.golang.org/genproto v0.0.0-20230526015343-6ee61e4f9d5f // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230526161137-0005af68ea54 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230526161137-0005af68ea54 // indirect google.golang.org/grpc v1.55.0 // indirect google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.1.7 // indirect diff --git a/go.sum b/go.sum index 324552aa..d0ca7f36 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/brianvoe/gofakeit/v6 v6.20.2 h1:FLloufuC7NcbHqDzVQ42CG9AKryS1gAGCRt8nQRsW+Y= github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= @@ -140,6 +144,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/gocql/gocql v1.6.0 h1:IdFdOTbnpbd0pDhl4REKQDM+Q0SzKXQ1Yh+YZZ8T/qU= +github.com/gocql/gocql v1.6.0/go.mod h1:3gM2c4D3AnkISwBxGnMMsS8Oy4y2lhbPRsH4xnJrHG8= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -180,6 +186,9 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -225,6 +234,8 @@ github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbsk github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= @@ -547,6 +558,10 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ= +github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc= +github.com/scylladb/gocqlx/v2 v2.8.0 h1:f/oIgoEPjKDKd+RIoeHqexsIQVIbalVmT+axwvUqQUg= +github.com/scylladb/gocqlx/v2 v2.8.0/go.mod h1:4/+cga34PVqjhgSoo5Nr2fX1MQIqZB5eCE5DK4xeDig= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= @@ -1018,6 +1033,8 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -- 2.51.2 From 8086f1376eef6ebe57d827d537354428bba59b7e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 30 Aug 2023 19:32:32 +0000 Subject: [PATCH 009/192] More scylla modeling --- Makefile | 7 +- cmd/netsync/playback.go | 302 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 284 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 8ede7724..e1f05a3a 100644 --- a/Makefile +++ b/Makefile @@ -122,7 +122,7 @@ run-netsync: .env ## Runs netsync for local dev .PHONY: netsync-playback netsync-playback: .env ## Runs netsync for local dev - go run ./cmd/netsync --worker-count 60 --out-dir ../netsync-out_2023_08_25 playback --scylla-nodes 127.0.0.1:9042 + go run ./cmd/netsync --worker-count 96 --out-dir ../netsync-out_2023_08_25 playback --scylla-nodes 127.0.0.1:9042 SCYLLA_VERSION := latest @@ -134,3 +134,8 @@ run-scylla: @docker pull scylladb/scylla:$(SCYLLA_VERSION) @docker run --name scylla -p 9042:9042 --cpuset-cpus=$(SCYLLA_CPU) --memory 1G --rm -d scylladb/scylla:$(SCYLLA_VERSION) @until docker exec scylla cqlsh -e "DESCRIBE SCHEMA"; do sleep 2; done + +.PHONY: stop-scylla +stop-scylla: + @echo "==> Stopping test instance of Scylla $(SCYLLA_VERSION)" + @docker stop scylla diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index a87768da..2f164188 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -2,7 +2,9 @@ package main import ( "context" + "encoding/json" "fmt" + "io/fs" "net/http" "os" "os/signal" @@ -20,6 +22,7 @@ import ( "github.com/ipfs/go-cid" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/scylladb/gocqlx/v2" + "github.com/scylladb/gocqlx/v2/qb" "github.com/scylladb/gocqlx/v2/table" "github.com/urfave/cli/v2" @@ -75,27 +78,139 @@ func (s *PlaybackState) Finish(repo string, state string) { } var postMetadata = table.Metadata{ - Name: "netsync.post", - Columns: []string{"did", "rkey", "content", "created_at"}, + Name: "netsync.posts", + Columns: []string{"did", "rkey", "display_name", "content", "facets", "created_at"}, PartKey: []string{"did"}, SortKey: []string{"rkey"}, } var postTable = table.New(postMetadata) type Post struct { + Did string + DisplayName string + Rkey string + Content string + Facets string + CreatedAt time.Time +} + +var followByActorMetadata = table.Metadata{ + Name: "netsync.follows_by_actor", + Columns: []string{"actor", "target", "created_at"}, + PartKey: []string{"actor"}, + SortKey: []string{"target"}, +} +var followByActorTable = table.New(followByActorMetadata) + +type FollowByActor struct { + Actor string + Target string + CreatedAt time.Time +} + +var followByTargetMetadata = table.Metadata{ + Name: "netsync.follows_by_target", + Columns: []string{"target", "actor", "created_at"}, + PartKey: []string{"target"}, + SortKey: []string{"actor"}, +} +var followByTargetTable = table.New(followByTargetMetadata) + +type FollowByTarget struct { + Target string + Actor string + CreatedAt time.Time +} + +var blockByActorMetadata = table.Metadata{ + Name: "netsync.blocks_by_actor", + Columns: []string{"actor", "target", "created_at"}, + PartKey: []string{"actor"}, + SortKey: []string{"target"}, +} +var blockByActorTable = table.New(blockByActorMetadata) + +type BlockByActor struct { + Actor string + Target string + CreatedAt time.Time +} + +var blockByTargetMetadata = table.Metadata{ + Name: "netsync.blocks_by_target", + Columns: []string{"target", "actor", "created_at"}, + PartKey: []string{"target"}, + SortKey: []string{"actor"}, +} +var blockByTargetTable = table.New(blockByTargetMetadata) + +type BlockByTarget struct { + Target string + Actor string + CreatedAt time.Time +} + +var likesMetadata = table.Metadata{ + Name: "netsync.likes", + Columns: []string{"did", "rkey", "subject", "created_at"}, + PartKey: []string{"did"}, + SortKey: []string{"rkey"}, +} +var likesTable = table.New(likesMetadata) + +type Likes struct { Did string Rkey string - Content string + Subject string CreatedAt time.Time } +var likeCountMetadata = table.Metadata{ + Name: "netsync.like_counts", + Columns: []string{"did", "nsid", "rkey", "count"}, + PartKey: []string{"did", "nsid"}, + SortKey: []string{"rkey"}, +} +var likeCountTable = table.New(likeCountMetadata) + +type LikeCount struct { + Did string + Nsid string + Rkey string + Count int64 +} + func (s *PlaybackState) SetupSchema() error { if err := s.ses.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS netsync WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };`); err != nil { return fmt.Errorf("failed to create keyspace: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.post (did text, rkey text, content text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { - return fmt.Errorf("failed to create post table: %w", err) + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, content text, facets text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + return fmt.Errorf("failed to create posts table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { + return fmt.Errorf("failed to create follows by actor table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_target (target text, actor text, created_at timestamp, PRIMARY KEY (target, actor));`); err != nil { + return fmt.Errorf("failed to create follows by target table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { + return fmt.Errorf("failed to create blocks by actor table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_target (target text, actor text, created_at timestamp, PRIMARY KEY (target, actor));`); err != nil { + return fmt.Errorf("failed to create blocks by target table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.likes (did text, rkey text, subject text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + return fmt.Errorf("failed to create likes table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.like_counts (did text, nsid text, rkey text, count counter, PRIMARY KEY ((did, nsid), rkey));`); err != nil { + return fmt.Errorf("failed to create like counts table: %w", err) } return nil @@ -150,17 +265,17 @@ func Playback(cctx *cli.Context) error { }() // Load all the repos from the out dir - err = filepath.Walk(state.outDir, func(path string, info os.FileInfo, err error) error { + err = filepath.WalkDir(state.outDir, func(path string, d fs.DirEntry, err error) error { if err != nil { return fmt.Errorf("failed to walk path: %w", err) } - if info.IsDir() { + if d.IsDir() { return nil } - state.EnqueuedRepos[info.Name()] = &RepoState{ - Repo: info.Name(), + state.EnqueuedRepos[d.Name()] = &RepoState{ + Repo: d.Name(), State: "enqueued", } @@ -278,16 +393,33 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return "", fmt.Errorf("failed to read repo from car: %w", err) } - r.ForEach(ctx, "", func(path string, nodeCid cid.Cid) error { - recordCid, rec, err := r.GetRecord(ctx, path) - if err != nil { - return fmt.Errorf("failed to get record: %w", err) + maxBatchSize := 1000 + + postBatch := s.ses.NewBatch(gocql.LoggedBatch) + postBatchSize := 0 + + followByActorBatch := s.ses.NewBatch(gocql.LoggedBatch) + followByActorBatchSize := 0 + + likeBatch := s.ses.NewBatch(gocql.LoggedBatch) + likeBatchSize := 0 + + displayName := "unknown" + + _, rec, err := r.GetRecord(ctx, "app.bsky.actor.profile/self") + if err == nil { + switch rec := rec.(type) { + case *bsky.ActorProfile: + if rec.DisplayName != nil { + displayName = *rec.DisplayName + } } + } - // Verify that the record cid matches the cid in the event - if recordCid != nodeCid { - log.Errorf("mismatch in record and op cid: %s != %s", recordCid, nodeCid) - return nil + err = r.ForEach(ctx, "", func(path string, _ cid.Cid) error { + _, rec, err := r.GetRecord(ctx, path) + if err != nil { + return fmt.Errorf("failed to get record: %w", err) } rkey := strings.Split(path, "/")[1] @@ -298,27 +430,113 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta s.textLen.Add(uint64(len(rec.Text))) recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) if err != nil { - log.Errorf("failed to parse created at: %w", err) + log.Errorf("failed to parse created at: %+v", err) return nil } insertPost := postTable.InsertQuery(s.ses) - insertPost.BindStruct(Post{ + + facets := "" + if rec.Facets != nil && len(rec.Facets) > 0 { + facetBytes, err := json.Marshal(rec.Facets) + if err != nil { + log.Errorf("failed to marshal facets: %+v", err) + return nil + } + facets = string(facetBytes) + } + + err = postBatch.BindStruct(insertPost, &Post{ + Did: did, + Rkey: rkey, + DisplayName: displayName, + Content: rec.Text, + Facets: facets, + CreatedAt: recCreatedAt, + }) + if err != nil { + log.Errorf("failed to bind post: %w", err) + return nil + } + postBatchSize++ + case *bsky.FeedLike: + log.Debugf("processing feed like: %s", rec.Subject.Uri) + recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) + if err != nil { + log.Errorf("failed to parse created at: %+v", err) + return nil + } + + insertLike := likesTable.InsertQuery(s.ses) + err = likeBatch.BindStruct(insertLike, &Likes{ Did: did, Rkey: rkey, - Content: rec.Text, + Subject: rec.Subject.Uri, CreatedAt: recCreatedAt, }) - if err := insertPost.ExecRelease(); err != nil { - log.Errorf("failed to insert post: %+v", err) + if err != nil { + log.Errorf("failed to bind like: %w", err) return nil } - case *bsky.FeedLike: - log.Debugf("processing feed like: %s", rec.Subject.Uri) + likeBatchSize++ + + // Don't batch like count because the partition key isn't consistent + subj := strings.TrimPrefix(rec.Subject.Uri, "at://") + subjParts := strings.Split(subj, "/") + if len(subjParts) != 3 { + log.Errorf("invalid subject: %s", rec.Subject.Uri) + return nil + } + + updateLikeCount := likeCountTable.UpdateBuilder(). + Add("count").Where(qb.Eq("did"), qb.Eq("nsid"), qb.Eq("rkey")).Query(s.ses). + BindStruct(&LikeCount{ + Did: subjParts[0], + Nsid: subjParts[1], + Rkey: subjParts[2], + Count: 1, + }) + + err = updateLikeCount.ExecRelease() + if err != nil { + log.Errorf("failed to exec like count: %w", err) + return nil + } + case *bsky.FeedRepost: log.Debugf("processing feed repost: %s", rec.Subject.Uri) case *bsky.GraphFollow: log.Debugf("processing graph follow: %s", rec.Subject) + recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) + if err != nil { + log.Errorf("failed to parse created at: %+v", err) + return nil + } + + insertFollowByActor := followByActorTable.InsertQuery(s.ses) + insertFollowByTarget := followByTargetTable.InsertQuery(s.ses) + + err = followByActorBatch.BindStruct(insertFollowByActor, &FollowByActor{ + Actor: did, + Target: rec.Subject, + CreatedAt: recCreatedAt, + }) + if err != nil { + log.Errorf("failed to bind follow by actor: %w", err) + return nil + } + followByActorBatchSize++ + + // Don't batch follow by target because the partition key isn't consistent + err = insertFollowByTarget.BindStruct(&FollowByTarget{ + Target: rec.Subject, + Actor: did, + CreatedAt: recCreatedAt, + }).Exec() + if err != nil { + log.Errorf("failed to exec follow by target: %w", err) + return nil + } case *bsky.GraphBlock: log.Debugf("processing graph block: %s", rec.Subject) case *bsky.ActorProfile: @@ -327,8 +545,44 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } + if postBatchSize >= maxBatchSize { + err = s.ses.ExecuteBatch(postBatch) + if err != nil { + log.Errorf("failed to execute batch: %w", err) + return nil + } + postBatch = s.ses.NewBatch(gocql.LoggedBatch) + postBatchSize = 0 + } + + if followByActorBatchSize >= maxBatchSize { + err = s.ses.ExecuteBatch(followByActorBatch) + if err != nil { + log.Errorf("failed to execute batch: %w", err) + } + followByActorBatch = s.ses.NewBatch(gocql.LoggedBatch) + followByActorBatchSize = 0 + } + return nil }) + if err != nil { + return "failed (repo foreach)", fmt.Errorf("failed to process repo: %w", err) + } + + if postBatchSize > 0 { + err = s.ses.ExecuteBatch(postBatch) + if err != nil { + return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + } + } + + if followByActorBatchSize > 0 { + err = s.ses.ExecuteBatch(followByActorBatch) + if err != nil { + return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + } + } return "finished", nil } -- 2.51.2 From 05360c08c18e9683e67143ca73b565fd020be14e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 30 Aug 2023 20:06:49 +0000 Subject: [PATCH 010/192] Update gocqlx --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 64ee553f..fbf8b5b7 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/polydawn/refmt v0.89.1-0.20221221234430-40501e09de1f github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 - github.com/scylladb/gocqlx/v2 v2.8.0 + github.com/scylladb/gocqlx/v2 v2.8.1-0.20230309105046-dec046bd85e6 github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.25.1 github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 diff --git a/go.sum b/go.sum index d0ca7f36..aacaa0cb 100644 --- a/go.sum +++ b/go.sum @@ -560,8 +560,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/scylladb/go-reflectx v1.0.1 h1:b917wZM7189pZdlND9PbIJ6NQxfDPfBvUaQ7cjj1iZQ= github.com/scylladb/go-reflectx v1.0.1/go.mod h1:rWnOfDIRWBGN0miMLIcoPt/Dhi2doCMZqwMCJ3KupFc= -github.com/scylladb/gocqlx/v2 v2.8.0 h1:f/oIgoEPjKDKd+RIoeHqexsIQVIbalVmT+axwvUqQUg= -github.com/scylladb/gocqlx/v2 v2.8.0/go.mod h1:4/+cga34PVqjhgSoo5Nr2fX1MQIqZB5eCE5DK4xeDig= +github.com/scylladb/gocqlx/v2 v2.8.1-0.20230309105046-dec046bd85e6 h1:f44elFEpX8jdEbOSHb4NCASZFGFOPEIt009XG/JCS90= +github.com/scylladb/gocqlx/v2 v2.8.1-0.20230309105046-dec046bd85e6/go.mod h1:4/+cga34PVqjhgSoo5Nr2fX1MQIqZB5eCE5DK4xeDig= github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -- 2.51.2 From 0c4e751d3f06c2d91ebcdeb9ecfbe1891ca57ff9 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 30 Aug 2023 23:28:18 +0000 Subject: [PATCH 011/192] Change keyspace replication --- cmd/netsync/playback.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 2f164188..06562a77 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -181,7 +181,7 @@ type LikeCount struct { } func (s *PlaybackState) SetupSchema() error { - if err := s.ses.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS netsync WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 };`); err != nil { + if err := s.ses.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS netsync WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 };`); err != nil { return fmt.Errorf("failed to create keyspace: %w", err) } -- 2.51.2 From d80006f79bf573e08a69d3541d91b57a76269734 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 00:19:04 +0000 Subject: [PATCH 012/192] Execute likes batches lol --- cmd/netsync/playback.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 06562a77..f4e453b3 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -564,6 +564,15 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta followByActorBatchSize = 0 } + if likeBatchSize >= maxBatchSize { + err = s.ses.ExecuteBatch(likeBatch) + if err != nil { + log.Errorf("failed to execute batch: %w", err) + } + likeBatch = s.ses.NewBatch(gocql.LoggedBatch) + likeBatchSize = 0 + } + return nil }) if err != nil { @@ -584,5 +593,12 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } + if likeBatchSize > 0 { + err = s.ses.ExecuteBatch(likeBatch) + if err != nil { + return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + } + } + return "finished", nil } -- 2.51.2 From 5d9fbec635a57f49b1ed74593f2aa8e3da64e69d Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 01:04:04 +0000 Subject: [PATCH 013/192] Sort repos by size before processing --- cmd/netsync/playback.go | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index f4e453b3..3b619ba4 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,6 +9,7 @@ import ( "os" "os/signal" "path/filepath" + "slices" "strings" "sync" "sync/atomic" @@ -216,6 +217,11 @@ func (s *PlaybackState) SetupSchema() error { return nil } +type dirEnt struct { + name string + size int64 +} + func Playback(cctx *cli.Context) error { ctx := cctx.Context ctx, cancel := context.WithCancel(ctx) @@ -264,6 +270,8 @@ func Playback(cctx *cli.Context) error { log.Info("metrics server shut down successfully") }() + dirEnts := []dirEnt{} + // Load all the repos from the out dir err = filepath.WalkDir(state.outDir, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -274,19 +282,41 @@ func Playback(cctx *cli.Context) error { return nil } - state.EnqueuedRepos[d.Name()] = &RepoState{ - Repo: d.Name(), - State: "enqueued", + finfo, err := d.Info() + if err != nil { + return fmt.Errorf("failed to get dir entry info: %w", err) } - enqueuedJobs.Inc() - + dirEnts = append(dirEnts, dirEnt{ + name: path, + size: finfo.Size(), + }) return nil }) if err != nil { return fmt.Errorf("failed to walk out dir: %w", err) } + // Sort the repos by size descending + slices.SortFunc(dirEnts, func(a, b dirEnt) int { + if a.size > b.size { + return -1 + } + if a.size < b.size { + return 1 + } + return 0 + }) + + // Enqueue the repos + for _, ent := range dirEnts { + state.EnqueuedRepos[ent.name] = &RepoState{ + Repo: ent.name, + State: "enqueued", + } + enqueuedJobs.Inc() + } + // Start workers for i := 0; i < state.workerCount; i++ { go state.worker(i) -- 2.51.2 From 13e43af60c3efc058a61c99ceaa77e5a4f80ed09 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 01:08:21 +0000 Subject: [PATCH 014/192] JK this is a map --- cmd/netsync/playback.go | 40 +++++----------------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 3b619ba4..f4e453b3 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,7 +9,6 @@ import ( "os" "os/signal" "path/filepath" - "slices" "strings" "sync" "sync/atomic" @@ -217,11 +216,6 @@ func (s *PlaybackState) SetupSchema() error { return nil } -type dirEnt struct { - name string - size int64 -} - func Playback(cctx *cli.Context) error { ctx := cctx.Context ctx, cancel := context.WithCancel(ctx) @@ -270,8 +264,6 @@ func Playback(cctx *cli.Context) error { log.Info("metrics server shut down successfully") }() - dirEnts := []dirEnt{} - // Load all the repos from the out dir err = filepath.WalkDir(state.outDir, func(path string, d fs.DirEntry, err error) error { if err != nil { @@ -282,41 +274,19 @@ func Playback(cctx *cli.Context) error { return nil } - finfo, err := d.Info() - if err != nil { - return fmt.Errorf("failed to get dir entry info: %w", err) + state.EnqueuedRepos[d.Name()] = &RepoState{ + Repo: d.Name(), + State: "enqueued", } - dirEnts = append(dirEnts, dirEnt{ - name: path, - size: finfo.Size(), - }) + enqueuedJobs.Inc() + return nil }) if err != nil { return fmt.Errorf("failed to walk out dir: %w", err) } - // Sort the repos by size descending - slices.SortFunc(dirEnts, func(a, b dirEnt) int { - if a.size > b.size { - return -1 - } - if a.size < b.size { - return 1 - } - return 0 - }) - - // Enqueue the repos - for _, ent := range dirEnts { - state.EnqueuedRepos[ent.name] = &RepoState{ - Repo: ent.name, - State: "enqueued", - } - enqueuedJobs.Inc() - } - // Start workers for i := 0; i < state.workerCount; i++ { go state.worker(i) -- 2.51.2 From c4232ed0e0ed2a0705f1d0aa452b7eb3afea2e7e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 01:14:22 +0000 Subject: [PATCH 015/192] Insert blocks --- cmd/netsync/playback.go | 67 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index f4e453b3..e034125e 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -96,14 +96,15 @@ type Post struct { var followByActorMetadata = table.Metadata{ Name: "netsync.follows_by_actor", - Columns: []string{"actor", "target", "created_at"}, + Columns: []string{"actor", "rkey", "target", "created_at"}, PartKey: []string{"actor"}, - SortKey: []string{"target"}, + SortKey: []string{"rkey"}, } var followByActorTable = table.New(followByActorMetadata) type FollowByActor struct { Actor string + Rkey string Target string CreatedAt time.Time } @@ -124,14 +125,15 @@ type FollowByTarget struct { var blockByActorMetadata = table.Metadata{ Name: "netsync.blocks_by_actor", - Columns: []string{"actor", "target", "created_at"}, + Columns: []string{"actor", "rkey", "target", "created_at"}, PartKey: []string{"actor"}, - SortKey: []string{"target"}, + SortKey: []string{"rkey"}, } var blockByActorTable = table.New(blockByActorMetadata) type BlockByActor struct { Actor string + Rkey string Target string CreatedAt time.Time } @@ -189,7 +191,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create posts table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { return fmt.Errorf("failed to create follows by actor table: %w", err) } @@ -197,7 +199,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create follows by target table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { return fmt.Errorf("failed to create blocks by actor table: %w", err) } @@ -401,6 +403,9 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta followByActorBatch := s.ses.NewBatch(gocql.LoggedBatch) followByActorBatchSize := 0 + blockByActorBatch := s.ses.NewBatch(gocql.LoggedBatch) + blockByActorBatchSize := 0 + likeBatch := s.ses.NewBatch(gocql.LoggedBatch) likeBatchSize := 0 @@ -518,6 +523,7 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta err = followByActorBatch.BindStruct(insertFollowByActor, &FollowByActor{ Actor: did, + Rkey: rkey, Target: rec.Subject, CreatedAt: recCreatedAt, }) @@ -532,13 +538,44 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta Target: rec.Subject, Actor: did, CreatedAt: recCreatedAt, - }).Exec() + }).ExecRelease() if err != nil { log.Errorf("failed to exec follow by target: %w", err) return nil } case *bsky.GraphBlock: log.Debugf("processing graph block: %s", rec.Subject) + recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) + if err != nil { + log.Errorf("failed to parse created at: %+v", err) + return nil + } + + insertBlockByActor := blockByActorTable.InsertQuery(s.ses) + insertBlockByTarget := blockByTargetTable.InsertQuery(s.ses) + + err = blockByActorBatch.BindStruct(insertBlockByActor, &BlockByActor{ + Actor: did, + Rkey: rkey, + Target: rec.Subject, + CreatedAt: recCreatedAt, + }) + if err != nil { + log.Errorf("failed to bind block by actor: %w", err) + return nil + } + blockByActorBatchSize++ + + // Don't batch block by target because the partition key isn't consistent + err = insertBlockByTarget.BindStruct(&BlockByTarget{ + Target: rec.Subject, + Actor: did, + CreatedAt: recCreatedAt, + }).ExecRelease() + if err != nil { + log.Errorf("failed to exec block by target: %w", err) + return nil + } case *bsky.ActorProfile: if rec.DisplayName != nil { log.Debugf("processing actor profile: %s", *rec.DisplayName) @@ -564,6 +601,15 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta followByActorBatchSize = 0 } + if blockByActorBatchSize >= maxBatchSize { + err = s.ses.ExecuteBatch(blockByActorBatch) + if err != nil { + log.Errorf("failed to execute batch: %w", err) + } + blockByActorBatch = s.ses.NewBatch(gocql.LoggedBatch) + blockByActorBatchSize = 0 + } + if likeBatchSize >= maxBatchSize { err = s.ses.ExecuteBatch(likeBatch) if err != nil { @@ -593,6 +639,13 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } + if blockByActorBatchSize > 0 { + err = s.ses.ExecuteBatch(blockByActorBatch) + if err != nil { + return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + } + } + if likeBatchSize > 0 { err = s.ses.ExecuteBatch(likeBatch) if err != nil { -- 2.51.2 From a68b8153d6dee55a5410e438641617cd57c8775c Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 01:17:20 +0000 Subject: [PATCH 016/192] Exit workers quickly --- cmd/netsync/playback.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index e034125e..3fdf6e48 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -422,6 +422,12 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } err = r.ForEach(ctx, "", func(path string, _ cid.Cid) error { + select { + case <-s.exit: + return fmt.Errorf("exiting") + default: + } + _, rec, err := r.GetRecord(ctx, path) if err != nil { return fmt.Errorf("failed to get record: %w", err) -- 2.51.2 From b956e03fcba76be2ec408d2530b7f9dacd8df15c Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 02:38:59 +0000 Subject: [PATCH 017/192] Write out replies table --- cmd/netsync/playback.go | 54 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 3fdf6e48..f9877d92 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -94,6 +94,22 @@ type Post struct { CreatedAt time.Time } +var repliesMetadata = table.Metadata{ + Name: "netsync.replies", + Columns: []string{"parent_did", "parent_rkey", "child_did", "child_rkey", "created_at"}, + PartKey: []string{"parent_did", "parent_rkey"}, + SortKey: []string{"child_did", "child_rkey"}, +} +var repliesTable = table.New(repliesMetadata) + +type Reply struct { + ParentDid string + ParentRkey string + ChildDid string + ChildRkey string + CreatedAt time.Time +} + var followByActorMetadata = table.Metadata{ Name: "netsync.follows_by_actor", Columns: []string{"actor", "rkey", "target", "created_at"}, @@ -449,7 +465,19 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta facets := "" if rec.Facets != nil && len(rec.Facets) > 0 { - facetBytes, err := json.Marshal(rec.Facets) + nonNilFacets := []*bsky.RichtextFacet{} + + // Filter out nil facets + for i, facet := range rec.Facets { + for _, feature := range facet.Features { + if feature.RichtextFacet_Link != nil || feature.RichtextFacet_Mention != nil { + nonNilFacets = append(nonNilFacets, rec.Facets[i]) + break + } + } + } + + facetBytes, err := json.Marshal(nonNilFacets) if err != nil { log.Errorf("failed to marshal facets: %+v", err) return nil @@ -470,6 +498,30 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } postBatchSize++ + + if rec.Reply != nil && rec.Reply.Parent != nil { + // at://did/app.bsky.feed.post/rkey + parentURI := rec.Reply.Parent.Uri + parentURI = strings.TrimPrefix(parentURI, "at://") + parentParts := strings.Split(parentURI, "/") + if len(parentParts) != 3 { + log.Errorf("invalid parent URI: %s", parentURI) + return nil + } + + insertReply := repliesTable.InsertQuery(s.ses) + err = insertReply.BindStruct(&Reply{ + ParentDid: parentParts[0], + ParentRkey: parentParts[2], + ChildDid: did, + ChildRkey: rkey, + CreatedAt: recCreatedAt, + }).ExecRelease() + if err != nil { + log.Errorf("failed to exec reply: %w", err) + return nil + } + } case *bsky.FeedLike: log.Debugf("processing feed like: %s", rec.Subject.Uri) recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) -- 2.51.2 From 5d18977bcb3bb4978341c5d6d9c9f92e34c9a1a5 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 02:39:34 +0000 Subject: [PATCH 018/192] create replies table --- cmd/netsync/playback.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index f9877d92..a35aeaa7 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -207,6 +207,10 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create posts table: %w", err) } + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.replies (parent_did text, parent_rkey text, child_did text, child_rkey text, created_at timestamp, PRIMARY KEY ((parent_did, parent_rkey), child_did, child_rkey));`); err != nil { + return fmt.Errorf("failed to create replies table: %w", err) + } + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { return fmt.Errorf("failed to create follows by actor table: %w", err) } -- 2.51.2 From 59bd154681c78a2bdd19515285de656660c1e5ae Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 03:05:27 +0000 Subject: [PATCH 019/192] Table typos --- cmd/netsync/playback.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index a35aeaa7..402908e3 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -211,7 +211,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create replies table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, rkey));`); err != nil { return fmt.Errorf("failed to create follows by actor table: %w", err) } @@ -219,7 +219,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create follows by target table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, target));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, rkey));`); err != nil { return fmt.Errorf("failed to create blocks by actor table: %w", err) } -- 2.51.2 From 9dc33d134eebbc1f510922868ad9df256e59d17e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 03:24:25 +0000 Subject: [PATCH 020/192] Add parent did and rkey to post if present --- cmd/netsync/playback.go | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 402908e3..a338e9a0 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -79,7 +79,7 @@ func (s *PlaybackState) Finish(repo string, state string) { var postMetadata = table.Metadata{ Name: "netsync.posts", - Columns: []string{"did", "rkey", "display_name", "content", "facets", "created_at"}, + Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "facets", "created_at"}, PartKey: []string{"did"}, SortKey: []string{"rkey"}, } @@ -89,6 +89,8 @@ type Post struct { Did string DisplayName string Rkey string + ParentDid string + ParentRkey string Content string Facets string CreatedAt time.Time @@ -203,7 +205,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create keyspace: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, content text, facets text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, facets text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { return fmt.Errorf("failed to create posts table: %w", err) } @@ -489,30 +491,40 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta facets = string(facetBytes) } - err = postBatch.BindStruct(insertPost, &Post{ + parentParts := []string{} + if rec.Reply != nil && rec.Reply.Parent != nil { + // at://did/app.bsky.feed.post/rkey + parentURI := rec.Reply.Parent.Uri + parentURI = strings.TrimPrefix(parentURI, "at://") + parentParts = strings.Split(parentURI, "/") + if len(parentParts) != 3 { + log.Errorf("invalid parent URI: %s", parentURI) + return nil + } + } + + post := Post{ Did: did, Rkey: rkey, DisplayName: displayName, Content: rec.Text, Facets: facets, CreatedAt: recCreatedAt, - }) + } + + if len(parentParts) > 0 { + post.ParentDid = parentParts[0] + post.ParentRkey = parentParts[2] + } + + err = postBatch.BindStruct(insertPost, &post) if err != nil { log.Errorf("failed to bind post: %w", err) return nil } postBatchSize++ - if rec.Reply != nil && rec.Reply.Parent != nil { - // at://did/app.bsky.feed.post/rkey - parentURI := rec.Reply.Parent.Uri - parentURI = strings.TrimPrefix(parentURI, "at://") - parentParts := strings.Split(parentURI, "/") - if len(parentParts) != 3 { - log.Errorf("invalid parent URI: %s", parentURI) - return nil - } - + if len(parentParts) > 0 { insertReply := repliesTable.InsertQuery(s.ses) err = insertReply.BindStruct(&Reply{ ParentDid: parentParts[0], -- 2.51.2 From fe8cbdd496d19aea6dc9371ffd4bca7d0efec0a7 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 03:48:02 +0000 Subject: [PATCH 021/192] Add self labels --- cmd/netsync/playback.go | 32 +++++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index a338e9a0..df518ea0 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -79,7 +79,7 @@ func (s *PlaybackState) Finish(repo string, state string) { var postMetadata = table.Metadata{ Name: "netsync.posts", - Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "facets", "created_at"}, + Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "facets", "self_labels", "created_at"}, PartKey: []string{"did"}, SortKey: []string{"rkey"}, } @@ -89,11 +89,15 @@ type Post struct { Did string DisplayName string Rkey string - ParentDid string - ParentRkey string - Content string - Facets string - CreatedAt time.Time + + ParentDid string + ParentRkey string + + Content string + Facets string + SelfLabels []string + + CreatedAt time.Time } var repliesMetadata = table.Metadata{ @@ -205,7 +209,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create keyspace: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, facets text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, facets text, self_labels list, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { return fmt.Errorf("failed to create posts table: %w", err) } @@ -491,6 +495,16 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta facets = string(facetBytes) } + selfLabels := []string{} + + if rec.Labels != nil && + rec.Labels.LabelDefs_SelfLabels != nil && + len(rec.Labels.LabelDefs_SelfLabels.Values) > 0 { + for _, label := range rec.Labels.LabelDefs_SelfLabels.Values { + selfLabels = append(selfLabels, label.Val) + } + } + parentParts := []string{} if rec.Reply != nil && rec.Reply.Parent != nil { // at://did/app.bsky.feed.post/rkey @@ -517,6 +531,10 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta post.ParentRkey = parentParts[2] } + if len(selfLabels) > 0 { + post.SelfLabels = selfLabels + } + err = postBatch.BindStruct(insertPost, &post) if err != nil { log.Errorf("failed to bind post: %w", err) -- 2.51.2 From 6cd30a59c7a072c1254340972a4f7f7b8456167b Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 03:52:17 +0000 Subject: [PATCH 022/192] Add embeds --- cmd/netsync/playback.go | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index df518ea0..1174d393 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -79,7 +79,7 @@ func (s *PlaybackState) Finish(repo string, state string) { var postMetadata = table.Metadata{ Name: "netsync.posts", - Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "facets", "self_labels", "created_at"}, + Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "embed", "facets", "self_labels", "created_at"}, PartKey: []string{"did"}, SortKey: []string{"rkey"}, } @@ -94,6 +94,7 @@ type Post struct { ParentRkey string Content string + Embed string Facets string SelfLabels []string @@ -209,7 +210,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create keyspace: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, facets text, self_labels list, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, embed text, facets text, self_labels list, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { return fmt.Errorf("failed to create posts table: %w", err) } @@ -471,7 +472,13 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } - insertPost := postTable.InsertQuery(s.ses) + post := Post{ + Did: did, + Rkey: rkey, + DisplayName: displayName, + Content: rec.Text, + CreatedAt: recCreatedAt, + } facets := "" if rec.Facets != nil && len(rec.Facets) > 0 { @@ -495,6 +502,16 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta facets = string(facetBytes) } + embed := "" + if rec.Embed != nil { + embedBytes, err := json.Marshal(rec.Embed) + if err != nil { + log.Errorf("failed to marshal embed: %+v", err) + return nil + } + embed = string(embedBytes) + } + selfLabels := []string{} if rec.Labels != nil && @@ -517,13 +534,12 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } - post := Post{ - Did: did, - Rkey: rkey, - DisplayName: displayName, - Content: rec.Text, - Facets: facets, - CreatedAt: recCreatedAt, + if facets != "" { + post.Facets = facets + } + + if embed != "" { + post.Embed = embed } if len(parentParts) > 0 { @@ -535,6 +551,7 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta post.SelfLabels = selfLabels } + insertPost := postTable.InsertQuery(s.ses) err = postBatch.BindStruct(insertPost, &post) if err != nil { log.Errorf("failed to bind post: %w", err) -- 2.51.2 From db95aedf827d466451b7ed9edda1bea3b90205e2 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 03:56:17 +0000 Subject: [PATCH 023/192] Filter empty embeds --- cmd/netsync/playback.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 1174d393..a8e1eaf3 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -504,12 +504,18 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta embed := "" if rec.Embed != nil { - embedBytes, err := json.Marshal(rec.Embed) - if err != nil { - log.Errorf("failed to marshal embed: %+v", err) - return nil + // Filter out empty embeds + if rec.Embed.EmbedExternal != nil || + rec.Embed.EmbedImages != nil || + rec.Embed.EmbedRecord != nil || + rec.Embed.EmbedRecordWithMedia != nil { + embedBytes, err := json.Marshal(rec.Embed) + if err != nil { + log.Errorf("failed to marshal embed: %+v", err) + return nil + } + embed = string(embedBytes) } - embed = string(embedBytes) } selfLabels := []string{} -- 2.51.2 From f4a932bab441de78ae7665d3d337e944301cc25e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:01:09 +0000 Subject: [PATCH 024/192] Add a test query for getpostthread --- cmd/netsync/main.go | 12 ++++ cmd/netsync/playback.go | 122 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 56261500..7e45d7fe 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -126,6 +126,18 @@ func main() { }, }, }, + { + Name: "query", + Usage: "run a test query against scylla", + Action: Query, + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "scylla-nodes", + Usage: "list of scylla nodes to connect to", + EnvVars: []string{"SCYLLA_NODES"}, + }, + }, + }, } app.Action = Netsync diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index a8e1eaf3..c23b4a45 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,6 +9,7 @@ import ( "os" "os/signal" "path/filepath" + "slices" "strings" "sync" "sync/atomic" @@ -245,6 +246,127 @@ func (s *PlaybackState) SetupSchema() error { return nil } +func Query(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + start := time.Now() + + cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) + session, err := gocqlx.WrapSession(cluster.CreateSession()) + if err != nil { + return fmt.Errorf("failed to create scylla session: %w", err) + } + + args := cctx.Args() + if args.Len() != 1 { + return fmt.Errorf("must provide a post URI") + } + postURI := args.First() + + // at://did/app.bsky.feed.post/rkey + postURI = strings.TrimPrefix(postURI, "at://") + postParts := strings.Split(postURI, "/") + if len(postParts) != 3 { + return fmt.Errorf("invalid post URI: %s", postURI) + } + + // Get the post + post := Post{ + Did: postParts[0], + Rkey: postParts[2], + } + err = postTable.GetQuery(session).BindStruct(&post).ExecRelease() + if err != nil { + return fmt.Errorf("failed to get post: %w", err) + } + + // Get the replies + replyRefs := []*Reply{} + err = repliesTable.SelectQuery(session).BindStruct(&Reply{ + ParentDid: postParts[0], + ParentRkey: postParts[2], + }).SelectRelease(&replyRefs) + if err != nil { + return fmt.Errorf("failed to get replies: %w", err) + } + + replies := []Post{} + lk := sync.Mutex{} + + // Resolve the replies as posts in parallel + var wg sync.WaitGroup + for _, replyRef := range replyRefs { + wg.Add(1) + go func(replyRef *Reply) { + defer wg.Done() + + reply := Post{ + Did: replyRef.ChildDid, + Rkey: replyRef.ChildRkey, + } + + err = postTable.GetQuery(session).BindStruct(&reply).ExecRelease() + if err != nil { + log.Errorf("failed to get reply: %w", err) + return + } + lk.Lock() + replies = append(replies, reply) + lk.Unlock() + }(replyRef) + } + + // Resolve the parent up to the root + parents := []Post{} + go func() { + wg.Add(1) + defer wg.Done() + parentDid := post.ParentDid + parentRkey := post.ParentRkey + for { + parent := Post{ + Did: parentDid, + Rkey: parentRkey, + } + err = postTable.GetQuery(session).BindStruct(&parent).ExecRelease() + if err != nil { + log.Errorf("failed to get parent: %w", err) + return + } + + parents = append(parents, parent) + + if parent.ParentDid == "" { + break + } + + parentDid = parent.ParentDid + parentRkey = parent.ParentRkey + } + }() + + wg.Wait() + + // Print the thread + p := message.NewPrinter(language.English) + log.Infof("post: %s", post.Content) + log.Infof("replies: %d", len(replies)) + for _, reply := range replies { + log.Infof(" %s", reply.Content) + } + slices.Reverse(parents) + log.Infof("parents: %d", len(parents)) + for _, parent := range parents { + log.Infof(" %s", parent.Content) + } + + log.Info(p.Sprintf("processed post with %d replies and resolved %d parents in %s", len(replies), len(parents), time.Since(start))) + + return nil +} + func Playback(cctx *cli.Context) error { ctx := cctx.Context ctx, cancel := context.WithCancel(ctx) -- 2.51.2 From 58554dd4464cd6d43cfd67d403d0684ec7ac0835 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:05:32 +0000 Subject: [PATCH 025/192] Loops --- cmd/netsync/playback.go | 51 ++++++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index c23b4a45..71e48507 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -297,8 +297,9 @@ func Query(cctx *cli.Context) error { // Resolve the replies as posts in parallel var wg sync.WaitGroup - for _, replyRef := range replyRefs { + for i := range replyRefs { wg.Add(1) + replyRef := replyRefs[i] go func(replyRef *Reply) { defer wg.Done() @@ -320,32 +321,34 @@ func Query(cctx *cli.Context) error { // Resolve the parent up to the root parents := []Post{} - go func() { - wg.Add(1) - defer wg.Done() - parentDid := post.ParentDid - parentRkey := post.ParentRkey - for { - parent := Post{ - Did: parentDid, - Rkey: parentRkey, - } - err = postTable.GetQuery(session).BindStruct(&parent).ExecRelease() - if err != nil { - log.Errorf("failed to get parent: %w", err) - return - } + if post.ParentDid != "" && post.ParentRkey != "" { + go func() { + wg.Add(1) + defer wg.Done() + parentDid := post.ParentDid + parentRkey := post.ParentRkey + for { + parent := Post{ + Did: parentDid, + Rkey: parentRkey, + } + err = postTable.GetQuery(session).BindStruct(&parent).ExecRelease() + if err != nil { + log.Errorf("failed to get parent: %w", err) + return + } - parents = append(parents, parent) + parents = append(parents, parent) - if parent.ParentDid == "" { - break - } + if parent.ParentDid == "" { + break + } - parentDid = parent.ParentDid - parentRkey = parent.ParentRkey - } - }() + parentDid = parent.ParentDid + parentRkey = parent.ParentRkey + } + }() + } wg.Wait() -- 2.51.2 From 6bf52946bea327bf993145409bdc4ba83658d87c Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:10:48 +0000 Subject: [PATCH 026/192] More deubgging --- cmd/netsync/playback.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 71e48507..12f694a4 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -277,13 +277,13 @@ func Query(cctx *cli.Context) error { Did: postParts[0], Rkey: postParts[2], } - err = postTable.GetQuery(session).BindStruct(&post).ExecRelease() + err = postTable.GetQuery(session).BindStruct(&post).Scan(&post) if err != nil { return fmt.Errorf("failed to get post: %w", err) } // Get the replies - replyRefs := []*Reply{} + replyRefs := []Reply{} err = repliesTable.SelectQuery(session).BindStruct(&Reply{ ParentDid: postParts[0], ParentRkey: postParts[2], @@ -300,7 +300,7 @@ func Query(cctx *cli.Context) error { for i := range replyRefs { wg.Add(1) replyRef := replyRefs[i] - go func(replyRef *Reply) { + go func(replyRef Reply) { defer wg.Done() reply := Post{ @@ -308,7 +308,7 @@ func Query(cctx *cli.Context) error { Rkey: replyRef.ChildRkey, } - err = postTable.GetQuery(session).BindStruct(&reply).ExecRelease() + err = postTable.GetQuery(session).BindStruct(&reply).Scan(&reply) if err != nil { log.Errorf("failed to get reply: %w", err) return -- 2.51.2 From 268ba8d7d866c1dc410ab11fa97b811e960551b7 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:12:30 +0000 Subject: [PATCH 027/192] foobasr --- cmd/netsync/playback.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 12f694a4..de436290 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -277,7 +277,7 @@ func Query(cctx *cli.Context) error { Did: postParts[0], Rkey: postParts[2], } - err = postTable.GetQuery(session).BindStruct(&post).Scan(&post) + err = postTable.GetQuery(session).BindStruct(&post).SelectRelease(&post) if err != nil { return fmt.Errorf("failed to get post: %w", err) } @@ -308,7 +308,7 @@ func Query(cctx *cli.Context) error { Rkey: replyRef.ChildRkey, } - err = postTable.GetQuery(session).BindStruct(&reply).Scan(&reply) + err = postTable.GetQuery(session).BindStruct(&reply).SelectRelease(&reply) if err != nil { log.Errorf("failed to get reply: %w", err) return @@ -332,7 +332,7 @@ func Query(cctx *cli.Context) error { Did: parentDid, Rkey: parentRkey, } - err = postTable.GetQuery(session).BindStruct(&parent).ExecRelease() + err = postTable.GetQuery(session).BindStruct(&parent).SelectRelease(&parent) if err != nil { log.Errorf("failed to get parent: %w", err) return -- 2.51.2 From 4de20099f76213e4def66921dbb7d6388ba1fcd8 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:13:03 +0000 Subject: [PATCH 028/192] Think i got it this time --- cmd/netsync/playback.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index de436290..dfcba7b8 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -277,7 +277,7 @@ func Query(cctx *cli.Context) error { Did: postParts[0], Rkey: postParts[2], } - err = postTable.GetQuery(session).BindStruct(&post).SelectRelease(&post) + err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) if err != nil { return fmt.Errorf("failed to get post: %w", err) } @@ -308,7 +308,7 @@ func Query(cctx *cli.Context) error { Rkey: replyRef.ChildRkey, } - err = postTable.GetQuery(session).BindStruct(&reply).SelectRelease(&reply) + err = postTable.GetQuery(session).BindStruct(&reply).GetRelease(&reply) if err != nil { log.Errorf("failed to get reply: %w", err) return @@ -332,7 +332,7 @@ func Query(cctx *cli.Context) error { Did: parentDid, Rkey: parentRkey, } - err = postTable.GetQuery(session).BindStruct(&parent).SelectRelease(&parent) + err = postTable.GetQuery(session).BindStruct(&parent).GetRelease(&parent) if err != nil { log.Errorf("failed to get parent: %w", err) return -- 2.51.2 From f025affd55d48bc63ab5c63b7760c92821da0f50 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:20:08 +0000 Subject: [PATCH 029/192] Debugging --- cmd/netsync/playback.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index dfcba7b8..6f764b6a 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -328,6 +328,7 @@ func Query(cctx *cli.Context) error { parentDid := post.ParentDid parentRkey := post.ParentRkey for { + log.Infof("getting parent: %s/%s", parentDid, parentRkey) parent := Post{ Did: parentDid, Rkey: parentRkey, -- 2.51.2 From 84104b4d13d39554b01192d1006343bcb282dcb4 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 05:26:24 +0000 Subject: [PATCH 030/192] I'm dumb lol --- cmd/netsync/playback.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 6f764b6a..3bfc6803 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -322,13 +322,12 @@ func Query(cctx *cli.Context) error { // Resolve the parent up to the root parents := []Post{} if post.ParentDid != "" && post.ParentRkey != "" { + wg.Add(1) go func() { - wg.Add(1) defer wg.Done() parentDid := post.ParentDid parentRkey := post.ParentRkey for { - log.Infof("getting parent: %s/%s", parentDid, parentRkey) parent := Post{ Did: parentDid, Rkey: parentRkey, -- 2.51.2 From 7f737d6f54a91440c81596f521c7e91a2daf0300 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 06:09:43 +0000 Subject: [PATCH 031/192] Bench experiment --- cmd/netsync/playback.go | 206 +++++++++++++++++++++++++--------------- 1 file changed, 127 insertions(+), 79 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 3bfc6803..3c98931e 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,7 +9,6 @@ import ( "os" "os/signal" "path/filepath" - "slices" "strings" "sync" "sync/atomic" @@ -27,6 +26,7 @@ import ( "github.com/scylladb/gocqlx/v2/table" "github.com/urfave/cli/v2" + "golang.org/x/sync/semaphore" "golang.org/x/text/language" "golang.org/x/text/message" ) @@ -251,8 +251,6 @@ func Query(cctx *cli.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() - start := time.Now() - cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) session, err := gocqlx.WrapSession(cluster.CreateSession()) if err != nil { @@ -272,100 +270,150 @@ func Query(cctx *cli.Context) error { return fmt.Errorf("invalid post URI: %s", postURI) } - // Get the post - post := Post{ - Did: postParts[0], - Rkey: postParts[2], - } - err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) - if err != nil { - return fmt.Errorf("failed to get post: %w", err) - } + numRuns := 500000 + maxConcurrent := 400 + sem := semaphore.NewWeighted(int64(maxConcurrent)) - // Get the replies - replyRefs := []Reply{} - err = repliesTable.SelectQuery(session).BindStruct(&Reply{ - ParentDid: postParts[0], - ParentRkey: postParts[2], - }).SelectRelease(&replyRefs) - if err != nil { - return fmt.Errorf("failed to get replies: %w", err) - } + totalRowsRead := atomic.Uint64{} - replies := []Post{} - lk := sync.Mutex{} + runtimes := make(chan time.Duration, numRuns) - // Resolve the replies as posts in parallel - var wg sync.WaitGroup - for i := range replyRefs { - wg.Add(1) - replyRef := replyRefs[i] - go func(replyRef Reply) { - defer wg.Done() + start := time.Now() - reply := Post{ - Did: replyRef.ChildDid, - Rkey: replyRef.ChildRkey, + // Run the query in numRuns goroutines + var pwg sync.WaitGroup + for i := 0; i < numRuns; i++ { + pwg.Add(1) + go func() error { + defer pwg.Done() + sem.Acquire(ctx, 1) + defer sem.Release(1) + iterStart := time.Now() + defer func() { + runtimes <- time.Since(iterStart) + }() + + // Get the post + post := Post{ + Did: postParts[0], + Rkey: postParts[2], + } + err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) + if err != nil { + return fmt.Errorf("failed to get post: %w", err) } - err = postTable.GetQuery(session).BindStruct(&reply).GetRelease(&reply) + totalRowsRead.Add(1) + + // Get the replies + replyRefs := []Reply{} + err = repliesTable.SelectQuery(session).BindStruct(&Reply{ + ParentDid: postParts[0], + ParentRkey: postParts[2], + }).SelectRelease(&replyRefs) if err != nil { - log.Errorf("failed to get reply: %w", err) - return + return fmt.Errorf("failed to get replies: %w", err) } - lk.Lock() - replies = append(replies, reply) - lk.Unlock() - }(replyRef) - } - - // Resolve the parent up to the root - parents := []Post{} - if post.ParentDid != "" && post.ParentRkey != "" { - wg.Add(1) - go func() { - defer wg.Done() - parentDid := post.ParentDid - parentRkey := post.ParentRkey - for { - parent := Post{ - Did: parentDid, - Rkey: parentRkey, - } - err = postTable.GetQuery(session).BindStruct(&parent).GetRelease(&parent) - if err != nil { - log.Errorf("failed to get parent: %w", err) - return - } - parents = append(parents, parent) + totalRowsRead.Add(uint64(len(replyRefs))) - if parent.ParentDid == "" { - break - } + replies := []Post{} + lk := sync.Mutex{} + + // Resolve the replies as posts in parallel + var wg sync.WaitGroup + for i := range replyRefs { + wg.Add(1) + replyRef := replyRefs[i] + go func(replyRef Reply) { + defer wg.Done() + + reply := Post{ + Did: replyRef.ChildDid, + Rkey: replyRef.ChildRkey, + } + + err = postTable.GetQuery(session).BindStruct(&reply).GetRelease(&reply) + if err != nil { + log.Errorf("failed to get reply: %+v", err) + return + } + lk.Lock() + replies = append(replies, reply) + lk.Unlock() + }(replyRef) + } + + totalRowsRead.Add(uint64(len(replies))) + + // Resolve the parent up to the root + parents := []Post{} + if post.ParentDid != "" && post.ParentRkey != "" { + wg.Add(1) + go func() { + defer wg.Done() + parentDid := post.ParentDid + parentRkey := post.ParentRkey + for { + parent := Post{ + Did: parentDid, + Rkey: parentRkey, + } + err = postTable.GetQuery(session).BindStruct(&parent).GetRelease(&parent) + if err != nil && err != gocql.ErrNotFound { + log.Errorf("failed to get parent: %+v", err) + return + } + + parents = append(parents, parent) + + if parent.ParentDid == "" { + break + } - parentDid = parent.ParentDid - parentRkey = parent.ParentRkey + parentDid = parent.ParentDid + parentRkey = parent.ParentRkey + } + }() } + + totalRowsRead.Add(uint64(len(parents))) + + wg.Wait() + return nil }() } - wg.Wait() + // // Print the thread + // p := message.NewPrinter(language.English) + // log.Debugf("post: %s", post.Content) + // log.Debugf("replies: %d", len(replies)) + // for _, reply := range replies { + // log.Debugf(" %s", reply.Content) + // } + // slices.Reverse(parents) + // log.Debugf("parents: %d", len(parents)) + // for _, parent := range parents { + // log.Debugf(" %s", parent.Content) + // } - // Print the thread - p := message.NewPrinter(language.English) - log.Infof("post: %s", post.Content) - log.Infof("replies: %d", len(replies)) - for _, reply := range replies { - log.Infof(" %s", reply.Content) - } - slices.Reverse(parents) - log.Infof("parents: %d", len(parents)) - for _, parent := range parents { - log.Infof(" %s", parent.Content) + // log.Info(p.Sprintf("processed post with %d replies and resolved %d parents in %s", len(replies), len(parents), time.Since(start))) + + // Wait for all the queries to finish + pwg.Wait() + clockTime := time.Since(start) + close(runtimes) + + // Calculate the average runtime + var total time.Duration + for runtime := range runtimes { + total += runtime } + avg := total / time.Duration(numRuns) + + p := message.NewPrinter(language.English) - log.Info(p.Sprintf("processed post with %d replies and resolved %d parents in %s", len(replies), len(parents), time.Since(start))) + log.Info(p.Sprintf("processed post %d times (%d total reads) in %s (avg: %s, total: %s)", numRuns, totalRowsRead.Load(), clockTime, avg, total)) return nil } -- 2.51.2 From 5c9d79e03da92db626b3b53fe39a0bb6e05b0304 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 22:14:37 +0000 Subject: [PATCH 032/192] Configurable scylla nodes --- Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index e1f05a3a..1e762fb0 100644 --- a/Makefile +++ b/Makefile @@ -120,13 +120,13 @@ sc-fire: # Fires supercollider run-netsync: .env ## Runs netsync for local dev go run ./cmd/netsync--checkout-limit 40 --worker-count 60 --out-dir ../netsync-out -.PHONY: netsync-playback -netsync-playback: .env ## Runs netsync for local dev - go run ./cmd/netsync --worker-count 96 --out-dir ../netsync-out_2023_08_25 playback --scylla-nodes 127.0.0.1:9042 - - SCYLLA_VERSION := latest SCYLLA_CPU := 0 +SCYLLA_NODES := 127.0.0.1:9042 + +.PHONY: netsync-playback +netsync-playback: .env ## Runs netsync for local dev + go run ./cmd/netsync --worker-count 96 --out-dir ../netsync-out_2023_08_25 playback --scylla-nodes $(SCYLLA_NODES) .PHONY: run-scylla run-scylla: -- 2.51.2 From 0070c57d5ec42565d1c79910c363fea32ec3a752 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 31 Aug 2023 22:24:03 +0000 Subject: [PATCH 033/192] Add reposts to layout --- cmd/netsync/playback.go | 102 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 3c98931e..8e620c34 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -184,7 +184,7 @@ var likesMetadata = table.Metadata{ } var likesTable = table.New(likesMetadata) -type Likes struct { +type Like struct { Did string Rkey string Subject string @@ -206,6 +206,36 @@ type LikeCount struct { Count int64 } +var repostsMetadata = table.Metadata{ + Name: "netsync.reposts", + Columns: []string{"did", "rkey", "subject", "created_at"}, + PartKey: []string{"did"}, + SortKey: []string{"rkey"}, +} +var repostsTable = table.New(repostsMetadata) + +type Repost struct { + Did string + Rkey string + Subject string + CreatedAt time.Time +} + +var repostCountMetadata = table.Metadata{ + Name: "netsync.repost_counts", + Columns: []string{"did", "nsid", "rkey", "count"}, + PartKey: []string{"did", "nsid"}, + SortKey: []string{"rkey"}, +} +var repostCountTable = table.New(repostCountMetadata) + +type RepostCount struct { + Did string + Nsid string + Rkey string + Count int64 +} + func (s *PlaybackState) SetupSchema() error { if err := s.ses.ExecStmt(`CREATE KEYSPACE IF NOT EXISTS netsync WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : 3 };`); err != nil { return fmt.Errorf("failed to create keyspace: %w", err) @@ -243,6 +273,14 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create like counts table: %w", err) } + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.reposts (did text, rkey text, subject text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + return fmt.Errorf("failed to create reposts table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.repost_counts (did text, nsid text, rkey text, count counter, PRIMARY KEY ((did, nsid), rkey));`); err != nil { + return fmt.Errorf("failed to create repost counts table: %w", err) + } + return nil } @@ -609,6 +647,9 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta likeBatch := s.ses.NewBatch(gocql.LoggedBatch) likeBatchSize := 0 + repostBatch := s.ses.NewBatch(gocql.LoggedBatch) + repostBatchSize := 0 + displayName := "unknown" _, rec, err := r.GetRecord(ctx, "app.bsky.actor.profile/self") @@ -761,7 +802,7 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } insertLike := likesTable.InsertQuery(s.ses) - err = likeBatch.BindStruct(insertLike, &Likes{ + err = likeBatch.BindStruct(insertLike, &Like{ Did: did, Rkey: rkey, Subject: rec.Subject.Uri, @@ -798,6 +839,47 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta case *bsky.FeedRepost: log.Debugf("processing feed repost: %s", rec.Subject.Uri) + recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) + if err != nil { + log.Errorf("failed to parse created at: %+v", err) + return nil + } + + insertRepost := repostsTable.InsertQuery(s.ses) + err = repostBatch.BindStruct(insertRepost, &Repost{ + Did: did, + Rkey: rkey, + Subject: rec.Subject.Uri, + CreatedAt: recCreatedAt, + }) + if err != nil { + log.Errorf("failed to bind repost: %w", err) + return nil + } + repostBatchSize++ + + // Don't batch repost count because the partition key isn't consistent + subj := strings.TrimPrefix(rec.Subject.Uri, "at://") + subjParts := strings.Split(subj, "/") + if len(subjParts) != 3 { + log.Errorf("invalid subject: %s", rec.Subject.Uri) + return nil + } + + updateRepostCount := repostCountTable.UpdateBuilder(). + Add("count").Where(qb.Eq("did"), qb.Eq("nsid"), qb.Eq("rkey")).Query(s.ses). + BindStruct(&RepostCount{ + Did: subjParts[0], + Nsid: subjParts[1], + Rkey: subjParts[2], + Count: 1, + }) + + err = updateRepostCount.ExecRelease() + if err != nil { + log.Errorf("failed to exec repost count: %w", err) + return nil + } case *bsky.GraphFollow: log.Debugf("processing graph follow: %s", rec.Subject) recCreatedAt, err := dateparse.ParseAny(rec.CreatedAt) @@ -907,6 +989,15 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta likeBatchSize = 0 } + if repostBatchSize >= maxBatchSize { + err = s.ses.ExecuteBatch(repostBatch) + if err != nil { + log.Errorf("failed to execute batch: %w", err) + } + repostBatch = s.ses.NewBatch(gocql.LoggedBatch) + repostBatchSize = 0 + } + return nil }) if err != nil { @@ -941,5 +1032,12 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } + if repostBatchSize > 0 { + err = s.ses.ExecuteBatch(repostBatch) + if err != nil { + return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + } + } + return "finished", nil } -- 2.51.2 From 6ee08d5d9befd3b758a107df65409bb9911e244b Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 01:35:16 +0000 Subject: [PATCH 034/192] Post windows --- cmd/netsync/playback.go | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 8e620c34..b5911f26 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -81,8 +81,7 @@ func (s *PlaybackState) Finish(repo string, state string) { var postMetadata = table.Metadata{ Name: "netsync.posts", Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "embed", "facets", "self_labels", "created_at"}, - PartKey: []string{"did"}, - SortKey: []string{"rkey"}, + PartKey: []string{"did", "rkey"}, } var postTable = table.New(postMetadata) @@ -102,6 +101,21 @@ type Post struct { CreatedAt time.Time } +var postWindowMetadata = table.Metadata{ + Name: "netsync.post_windows", + Columns: []string{"did", "window", "rkey", "created_at"}, + PartKey: []string{"did", "window"}, + SortKey: []string{"created_at"}, +} +var postWindowTable = table.New(postWindowMetadata) + +type PostWindow struct { + Did string + Window string + Rkey string + CreatedAt time.Time +} + var repliesMetadata = table.Metadata{ Name: "netsync.replies", Columns: []string{"parent_did", "parent_rkey", "child_did", "child_rkey", "created_at"}, @@ -241,10 +255,14 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create keyspace: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, embed text, facets text, self_labels list, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, embed text, facets text, self_labels list, created_at timestamp, PRIMARY KEY ((did, rkey)));`); err != nil { return fmt.Errorf("failed to create posts table: %w", err) } + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.post_windows (did text, window text, rkey text, created_at timestamp, PRIMARY KEY ((did, window), created_at));`); err != nil { + return fmt.Errorf("failed to create post windows table: %w", err) + } + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.replies (parent_did text, parent_rkey text, child_did text, child_rkey text, created_at timestamp, PRIMARY KEY ((parent_did, parent_rkey), child_did, child_rkey));`); err != nil { return fmt.Errorf("failed to create replies table: %w", err) } @@ -779,6 +797,20 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } postBatchSize++ + // Insert into post windows using the day as the window + window := recCreatedAt.Format("2006-01-02") + insertPostWindow := postWindowTable.InsertQuery(s.ses) + err = insertPostWindow.BindStruct(&PostWindow{ + Did: did, + Window: window, + Rkey: rkey, + CreatedAt: recCreatedAt, + }).ExecRelease() + if err != nil { + log.Errorf("failed to exec post window: %w", err) + return nil + } + if len(parentParts) > 0 { insertReply := repliesTable.InsertQuery(s.ses) err = insertReply.BindStruct(&Reply{ -- 2.51.2 From 68015510e430f2bc49a4250f513be8c83b13ad70 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 01:39:01 +0000 Subject: [PATCH 035/192] Remove outdated static column --- cmd/netsync/playback.go | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index b5911f26..0fb1d3af 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -80,15 +80,14 @@ func (s *PlaybackState) Finish(repo string, state string) { var postMetadata = table.Metadata{ Name: "netsync.posts", - Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "display_name", "content", "embed", "facets", "self_labels", "created_at"}, + Columns: []string{"did", "rkey", "parent_did", "parent_rkey", "content", "embed", "facets", "self_labels", "created_at"}, PartKey: []string{"did", "rkey"}, } var postTable = table.New(postMetadata) type Post struct { - Did string - DisplayName string - Rkey string + Did string + Rkey string ParentDid string ParentRkey string @@ -255,7 +254,7 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create keyspace: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, display_name text static, rkey text, parent_did text, parent_rkey text, content text, embed text, facets text, self_labels list, created_at timestamp, PRIMARY KEY ((did, rkey)));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts (did text, rkey text, parent_did text, parent_rkey text, content text, embed text, facets text, self_labels list, created_at timestamp, PRIMARY KEY ((did, rkey)));`); err != nil { return fmt.Errorf("failed to create posts table: %w", err) } @@ -668,18 +667,6 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta repostBatch := s.ses.NewBatch(gocql.LoggedBatch) repostBatchSize := 0 - displayName := "unknown" - - _, rec, err := r.GetRecord(ctx, "app.bsky.actor.profile/self") - if err == nil { - switch rec := rec.(type) { - case *bsky.ActorProfile: - if rec.DisplayName != nil { - displayName = *rec.DisplayName - } - } - } - err = r.ForEach(ctx, "", func(path string, _ cid.Cid) error { select { case <-s.exit: @@ -705,11 +692,10 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } post := Post{ - Did: did, - Rkey: rkey, - DisplayName: displayName, - Content: rec.Text, - CreatedAt: recCreatedAt, + Did: did, + Rkey: rkey, + Content: rec.Text, + CreatedAt: recCreatedAt, } facets := "" -- 2.51.2 From ecfddbfd26aea28dcb61c6fb3b9d2522ff8b9303 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 01:40:48 +0000 Subject: [PATCH 036/192] Remove post batches now --- cmd/netsync/playback.go | 23 +---------------------- 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 0fb1d3af..cfedd650 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -652,9 +652,6 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta maxBatchSize := 1000 - postBatch := s.ses.NewBatch(gocql.LoggedBatch) - postBatchSize := 0 - followByActorBatch := s.ses.NewBatch(gocql.LoggedBatch) followByActorBatchSize := 0 @@ -776,12 +773,11 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } insertPost := postTable.InsertQuery(s.ses) - err = postBatch.BindStruct(insertPost, &post) + err = insertPost.BindStruct(&post).ExecRelease() if err != nil { log.Errorf("failed to bind post: %w", err) return nil } - postBatchSize++ // Insert into post windows using the day as the window window := recCreatedAt.Format("2006-01-02") @@ -970,16 +966,6 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } - if postBatchSize >= maxBatchSize { - err = s.ses.ExecuteBatch(postBatch) - if err != nil { - log.Errorf("failed to execute batch: %w", err) - return nil - } - postBatch = s.ses.NewBatch(gocql.LoggedBatch) - postBatchSize = 0 - } - if followByActorBatchSize >= maxBatchSize { err = s.ses.ExecuteBatch(followByActorBatch) if err != nil { @@ -1022,13 +1008,6 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return "failed (repo foreach)", fmt.Errorf("failed to process repo: %w", err) } - if postBatchSize > 0 { - err = s.ses.ExecuteBatch(postBatch) - if err != nil { - return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) - } - } - if followByActorBatchSize > 0 { err = s.ses.ExecuteBatch(followByActorBatch) if err != nil { -- 2.51.2 From be565e2c0070bfa509ac86556c94f3ac601fc781 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 03:37:40 +0000 Subject: [PATCH 037/192] Testing querying all posts for a user by window --- cmd/netsync/main.go | 12 +++++ cmd/netsync/playback.go | 112 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index 7e45d7fe..cc0a4e08 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -138,6 +138,18 @@ func main() { }, }, }, + { + Name: "getPostsForUser", + Usage: "run a test query against scylla", + Action: GetPostsForUser, + Flags: []cli.Flag{ + &cli.StringSliceFlag{ + Name: "scylla-nodes", + Usage: "list of scylla nodes to connect to", + EnvVars: []string{"SCYLLA_NODES"}, + }, + }, + }, } app.Action = Netsync diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index cfedd650..62093951 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -301,6 +301,118 @@ func (s *PlaybackState) SetupSchema() error { return nil } +func GetPostsForUser(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) + session, err := gocqlx.WrapSession(cluster.CreateSession()) + if err != nil { + return fmt.Errorf("failed to create scylla session: %w", err) + } + + args := cctx.Args() + if args.Len() != 1 { + return fmt.Errorf("must provide a did") + } + + did := args.First() + + // Posts are stored in windows by day, so we need to query for all the windows in parallel since Jan 1, 2023 + // and then query for all the posts in each window + + maxParallel := 500_000 + sem := semaphore.NewWeighted(int64(maxParallel)) + + // Compute window names + windowRangeStart := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + windowRangeEnd := time.Now().UTC() + windowNames := []string{} + for windowRangeStart.Before(windowRangeEnd) { + windowNames = append(windowNames, windowRangeStart.Format("2006-01-02")) + windowRangeStart = windowRangeStart.Add(24 * time.Hour) + } + + start := time.Now() + + // Query for all the posts in each window in parallel + var wg sync.WaitGroup + postWindows := []PostWindow{} + postWindowsLk := sync.Mutex{} + for _, windowName := range windowNames { + wg.Add(1) + go func(windowName string) { + defer wg.Done() + + sem.Acquire(ctx, 1) + defer sem.Release(1) + + postWindow := PostWindow{ + Did: did, + Window: windowName, + } + + windows := []PostWindow{} + + err = postWindowTable.SelectQuery(session).BindStruct(&postWindow).SelectRelease(&windows) + if err != nil { + log.Errorf("failed to get post windows: %+v", err) + return + } + + postWindowsLk.Lock() + postWindows = append(postWindows, windows...) + postWindowsLk.Unlock() + }(windowName) + } + + wg.Wait() + + // Query for all the posts in each window in parallel + var wg2 sync.WaitGroup + postsLk := sync.Mutex{} + posts := []Post{} + for _, postWindow := range postWindows { + wg2.Add(1) + go func(postWindow PostWindow) { + defer wg2.Done() + + sem.Acquire(ctx, 1) + defer sem.Release(1) + + post := Post{ + Did: postWindow.Did, + Rkey: postWindow.Rkey, + } + + err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) + if err != nil { + log.Errorf("failed to get post: %+v", err) + return + } + + postsLk.Lock() + posts = append(posts, post) + postsLk.Unlock() + }(postWindow) + } + + wg2.Wait() + + end := time.Now() + + // Print the posts + p := message.NewPrinter(language.English) + for _, post := range posts { + log.Info(p.Sprintf("post: %s", post.Content)) + } + + log.Warn(p.Sprintf("got %d posts in %s", len(posts), end.Sub(start))) + + return nil +} + func Query(cctx *cli.Context) error { ctx := cctx.Context ctx, cancel := context.WithCancel(ctx) -- 2.51.2 From 3706a1ba42187f58570413a7be541d6ed0331f9a Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 03:41:55 +0000 Subject: [PATCH 038/192] Sort and limit results --- cmd/netsync/playback.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 62093951..240f9350 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,6 +9,7 @@ import ( "os" "os/signal" "path/filepath" + "slices" "strings" "sync" "sync/atomic" @@ -322,6 +323,7 @@ func GetPostsForUser(cctx *cli.Context) error { // Posts are stored in windows by day, so we need to query for all the windows in parallel since Jan 1, 2023 // and then query for all the posts in each window + limit := 500 maxParallel := 500_000 sem := semaphore.NewWeighted(int64(maxParallel)) @@ -369,6 +371,21 @@ func GetPostsForUser(cctx *cli.Context) error { wg.Wait() + // Sort the post windows by created at + slices.SortFunc(postWindows, func(a, b PostWindow) int { + if a.CreatedAt.Before(b.CreatedAt) { + return -1 + } else if a.CreatedAt.After(b.CreatedAt) { + return 1 + } + return 0 + }) + + // Limit the number of post windows + if len(postWindows) > limit { + postWindows = postWindows[:limit] + } + // Query for all the posts in each window in parallel var wg2 sync.WaitGroup postsLk := sync.Mutex{} -- 2.51.2 From 14cfdd404a7ab859332151e38d71ceb02bd7a1e8 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 05:14:53 +0000 Subject: [PATCH 039/192] Attempt using select-where-in and doing some cluster-size fanout --- cmd/netsync/playback.go | 72 +++++++++++++---------------------------- 1 file changed, 23 insertions(+), 49 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 240f9350..1a0786f9 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -320,15 +320,12 @@ func GetPostsForUser(cctx *cli.Context) error { did := args.First() - // Posts are stored in windows by day, so we need to query for all the windows in parallel since Jan 1, 2023 - // and then query for all the posts in each window - limit := 500 maxParallel := 500_000 sem := semaphore.NewWeighted(int64(maxParallel)) // Compute window names - windowRangeStart := time.Date(2023, 1, 1, 0, 0, 0, 0, time.UTC) + windowRangeStart := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC) windowRangeEnd := time.Now().UTC() windowNames := []string{} for windowRangeStart.Before(windowRangeEnd) { @@ -338,52 +335,18 @@ func GetPostsForUser(cctx *cli.Context) error { start := time.Now() - // Query for all the posts in each window in parallel - var wg sync.WaitGroup + // Query for all the posts in each window + // Note we can't query more than 100 partitions at a time using an IN clause postWindows := []PostWindow{} - postWindowsLk := sync.Mutex{} - for _, windowName := range windowNames { - wg.Add(1) - go func(windowName string) { - defer wg.Done() - - sem.Acquire(ctx, 1) - defer sem.Release(1) - - postWindow := PostWindow{ - Did: did, - Window: windowName, - } - - windows := []PostWindow{} - - err = postWindowTable.SelectQuery(session).BindStruct(&postWindow).SelectRelease(&windows) - if err != nil { - log.Errorf("failed to get post windows: %+v", err) - return - } - - postWindowsLk.Lock() - postWindows = append(postWindows, windows...) - postWindowsLk.Unlock() - }(windowName) - } - - wg.Wait() - - // Sort the post windows by created at - slices.SortFunc(postWindows, func(a, b PostWindow) int { - if a.CreatedAt.Before(b.CreatedAt) { - return -1 - } else if a.CreatedAt.After(b.CreatedAt) { - return 1 - } - return 0 - }) - - // Limit the number of post windows - if len(postWindows) > limit { - postWindows = postWindows[:limit] + err = qb.Select(postWindowMetadata.Name). + Where(qb.Eq("did"), qb.In("window")).OrderBy("created_at", qb.DESC). + Limit(uint(limit)).Query(session).BindMap(qb.M{ + "did": did, + "window": windowNames, + }).PageSize(-1).SelectRelease(&postWindows) + if err != nil { + log.Errorw("failed to get post windows", "err", err) + return nil } // Query for all the posts in each window in parallel @@ -419,6 +382,17 @@ func GetPostsForUser(cctx *cli.Context) error { end := time.Now() + // Sort the posts descending by created_at + slices.SortFunc(posts, func(a, b Post) int { + if a.CreatedAt.Before(b.CreatedAt) { + return 1 + } + if a.CreatedAt.After(b.CreatedAt) { + return -1 + } + return 0 + }) + // Print the posts p := message.NewPrinter(language.English) for _, post := range posts { -- 2.51.2 From 3d1ecfed3dfd603ba71aeb3ed742c0e6def3dc17 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 05:30:47 +0000 Subject: [PATCH 040/192] Parallelizing --- cmd/netsync/playback.go | 136 +++++++++++++++++++++++----------------- 1 file changed, 79 insertions(+), 57 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 1a0786f9..644b516e 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,7 +9,6 @@ import ( "os" "os/signal" "path/filepath" - "slices" "strings" "sync" "sync/atomic" @@ -321,8 +320,18 @@ func GetPostsForUser(cctx *cli.Context) error { did := args.First() limit := 500 - maxParallel := 500_000 - sem := semaphore.NewWeighted(int64(maxParallel)) + + numRuns := 64 + maxConcurrent := 200 + maxByKeyConcurrent := 20000 + sem := semaphore.NewWeighted(int64(maxConcurrent)) + semByKey := semaphore.NewWeighted(int64(maxByKeyConcurrent)) + + totalRowsRead := atomic.Uint64{} + + runtimes := make(chan time.Duration, numRuns) + + start := time.Now() // Compute window names windowRangeStart := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC) @@ -333,73 +342,86 @@ func GetPostsForUser(cctx *cli.Context) error { windowRangeStart = windowRangeStart.Add(24 * time.Hour) } - start := time.Now() - - // Query for all the posts in each window - // Note we can't query more than 100 partitions at a time using an IN clause - postWindows := []PostWindow{} - err = qb.Select(postWindowMetadata.Name). - Where(qb.Eq("did"), qb.In("window")).OrderBy("created_at", qb.DESC). - Limit(uint(limit)).Query(session).BindMap(qb.M{ - "did": did, - "window": windowNames, - }).PageSize(-1).SelectRelease(&postWindows) - if err != nil { - log.Errorw("failed to get post windows", "err", err) - return nil - } - - // Query for all the posts in each window in parallel - var wg2 sync.WaitGroup - postsLk := sync.Mutex{} - posts := []Post{} - for _, postWindow := range postWindows { - wg2.Add(1) - go func(postWindow PostWindow) { - defer wg2.Done() - + // Run the query in numRuns goroutines + var pwg sync.WaitGroup + for i := 0; i < numRuns; i++ { + pwg.Add(1) + go func() error { + defer pwg.Done() sem.Acquire(ctx, 1) defer sem.Release(1) + iterStart := time.Now() + defer func() { + runtimes <- time.Since(iterStart) + }() - post := Post{ - Did: postWindow.Did, - Rkey: postWindow.Rkey, + // Query for all the posts in each window + // Note we can't query more than 100 partitions at a time using an IN clause + postWindows := []PostWindow{} + err = qb.Select(postWindowMetadata.Name). + Where(qb.Eq("did"), qb.In("window")).OrderBy("created_at", qb.DESC). + Limit(uint(limit)).Query(session).BindMap(qb.M{ + "did": did, + "window": windowNames, + }).PageSize(-1).SelectRelease(&postWindows) + if err != nil { + log.Errorw("failed to get post windows", "err", err) + return nil } - err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) - if err != nil { - log.Errorf("failed to get post: %+v", err) - return + totalRowsRead.Add(uint64(len(postWindows))) + + // Query for all the posts in each window in parallel + var wg2 sync.WaitGroup + postsLk := sync.Mutex{} + posts := []Post{} + for _, postWindow := range postWindows { + wg2.Add(1) + go func(postWindow PostWindow) { + defer wg2.Done() + + semByKey.Acquire(ctx, 1) + defer semByKey.Release(1) + + post := Post{ + Did: postWindow.Did, + Rkey: postWindow.Rkey, + } + + err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) + if err != nil { + log.Errorf("failed to get post: %+v", err) + return + } + + postsLk.Lock() + posts = append(posts, post) + postsLk.Unlock() + }(postWindow) } + wg2.Wait() - postsLk.Lock() - posts = append(posts, post) - postsLk.Unlock() - }(postWindow) - } + totalRowsRead.Add(uint64(len(posts))) - wg2.Wait() + return nil + }() + } - end := time.Now() + // Wait for all the queries to finish + pwg.Wait() + clockTime := time.Since(start) + close(runtimes) - // Sort the posts descending by created_at - slices.SortFunc(posts, func(a, b Post) int { - if a.CreatedAt.Before(b.CreatedAt) { - return 1 - } - if a.CreatedAt.After(b.CreatedAt) { - return -1 - } - return 0 - }) + // Calculate the average runtime + var total time.Duration + for runtime := range runtimes { + total += runtime + } + avg := total / time.Duration(numRuns) - // Print the posts p := message.NewPrinter(language.English) - for _, post := range posts { - log.Info(p.Sprintf("post: %s", post.Content)) - } - log.Warn(p.Sprintf("got %d posts in %s", len(posts), end.Sub(start))) + log.Info(p.Sprintf("got post page %d times (%d total reads) in %s (avg: %s, total: %s)", numRuns, totalRowsRead.Load(), clockTime, avg, total)) return nil } -- 2.51.2 From ed52fadc245c64f09db23011e80d84c6d1e77441 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 1 Sep 2023 05:49:44 +0000 Subject: [PATCH 041/192] Querying smarter... --- cmd/netsync/playback.go | 68 +++++++++++++++++++++++++++++------------ 1 file changed, 48 insertions(+), 20 deletions(-) diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index 644b516e..a827f2a3 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,6 +9,7 @@ import ( "os" "os/signal" "path/filepath" + "slices" "strings" "sync" "sync/atomic" @@ -321,11 +322,9 @@ func GetPostsForUser(cctx *cli.Context) error { limit := 500 - numRuns := 64 - maxConcurrent := 200 - maxByKeyConcurrent := 20000 + numRuns := 2000 + maxConcurrent := 40 sem := semaphore.NewWeighted(int64(maxConcurrent)) - semByKey := semaphore.NewWeighted(int64(maxByKeyConcurrent)) totalRowsRead := atomic.Uint64{} @@ -334,7 +333,7 @@ func GetPostsForUser(cctx *cli.Context) error { start := time.Now() // Compute window names - windowRangeStart := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC) + windowRangeStart := time.Date(2023, 8, 1, 0, 0, 0, 0, time.UTC) windowRangeEnd := time.Now().UTC() windowNames := []string{} for windowRangeStart.Before(windowRangeEnd) { @@ -355,20 +354,53 @@ func GetPostsForUser(cctx *cli.Context) error { runtimes <- time.Since(iterStart) }() - // Query for all the posts in each window - // Note we can't query more than 100 partitions at a time using an IN clause + // Query for all the posts in each window in parallel + var wg sync.WaitGroup postWindows := []PostWindow{} - err = qb.Select(postWindowMetadata.Name). - Where(qb.Eq("did"), qb.In("window")).OrderBy("created_at", qb.DESC). - Limit(uint(limit)).Query(session).BindMap(qb.M{ - "did": did, - "window": windowNames, - }).PageSize(-1).SelectRelease(&postWindows) - if err != nil { - log.Errorw("failed to get post windows", "err", err) - return nil + postWindowsLk := sync.Mutex{} + for _, windowName := range windowNames { + wg.Add(1) + go func(windowName string) { + defer wg.Done() + + postWindow := PostWindow{ + Did: did, + Window: windowName, + } + + windows := []PostWindow{} + + err = qb.Select(postWindowMetadata.Name). + Where(qb.Eq("did"), qb.Eq("window")). + OrderBy("created_at", qb.DESC).Limit(uint(limit)). + Query(session).BindStruct(&postWindow).SelectRelease(&windows) + if err != nil { + log.Errorf("failed to get post windows: %+v", err) + return + } + + postWindowsLk.Lock() + postWindows = append(postWindows, windows...) + postWindowsLk.Unlock() + }(windowName) } + wg.Wait() + + // log.Infow("got post windows", "count", len(postWindows), "duration", time.Since(iterStart)) + + // Sort the post windows by created at and limit + slices.SortFunc(postWindows, func(a, b PostWindow) int { + if a.CreatedAt.Before(b.CreatedAt) { + return -1 + } + if a.CreatedAt.After(b.CreatedAt) { + return 1 + } + return 0 + }) + postWindows = postWindows[:limit] + totalRowsRead.Add(uint64(len(postWindows))) // Query for all the posts in each window in parallel @@ -379,10 +411,6 @@ func GetPostsForUser(cctx *cli.Context) error { wg2.Add(1) go func(postWindow PostWindow) { defer wg2.Done() - - semByKey.Acquire(ctx, 1) - defer semByKey.Release(1) - post := Post{ Did: postWindow.Did, Rkey: postWindow.Rkey, -- 2.51.2 From 0c0241a7b5079c5ed0df23f138369626bd1125fa Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 7 Sep 2023 02:37:54 +0000 Subject: [PATCH 042/192] Rev on schema --- cmd/netsync/playback.go | 686 +++++++++++++--------------------------- cmd/netsync/query.go | 230 ++++++++++++++ 2 files changed, 453 insertions(+), 463 deletions(-) create mode 100644 cmd/netsync/query.go diff --git a/cmd/netsync/playback.go b/cmd/netsync/playback.go index a827f2a3..abb43afe 100644 --- a/cmd/netsync/playback.go +++ b/cmd/netsync/playback.go @@ -9,7 +9,6 @@ import ( "os" "os/signal" "path/filepath" - "slices" "strings" "sync" "sync/atomic" @@ -27,7 +26,6 @@ import ( "github.com/scylladb/gocqlx/v2/table" "github.com/urfave/cli/v2" - "golang.org/x/sync/semaphore" "golang.org/x/text/language" "golang.org/x/text/message" ) @@ -101,17 +99,16 @@ type Post struct { CreatedAt time.Time } -var postWindowMetadata = table.Metadata{ - Name: "netsync.post_windows", - Columns: []string{"did", "window", "rkey", "created_at"}, - PartKey: []string{"did", "window"}, +var postsByDIDMetadata = table.Metadata{ + Name: "netsync.posts_by_did", + Columns: []string{"did", "rkey", "created_at"}, + PartKey: []string{"did"}, SortKey: []string{"created_at"}, } -var postWindowTable = table.New(postWindowMetadata) +var postsByDIDTable = table.New(postsByDIDMetadata) -type PostWindow struct { +type PostByDID struct { Did string - Window string Rkey string CreatedAt time.Time } @@ -132,17 +129,29 @@ type Reply struct { CreatedAt time.Time } +var followsMetadata = table.Metadata{ + Name: "netsync.follows", + Columns: []string{"actor", "rkey", "target", "created_at"}, + PartKey: []string{"actor", "rkey"}, +} +var followsTable = table.New(followsMetadata) + +type Follow struct { + Actor string + Rkey string + Target string + CreatedAt time.Time +} + var followByActorMetadata = table.Metadata{ Name: "netsync.follows_by_actor", - Columns: []string{"actor", "rkey", "target", "created_at"}, - PartKey: []string{"actor"}, - SortKey: []string{"rkey"}, + Columns: []string{"actor", "target", "created_at"}, + PartKey: []string{"actor", "target"}, } var followByActorTable = table.New(followByActorMetadata) type FollowByActor struct { Actor string - Rkey string Target string CreatedAt time.Time } @@ -150,8 +159,7 @@ type FollowByActor struct { var followByTargetMetadata = table.Metadata{ Name: "netsync.follows_by_target", Columns: []string{"target", "actor", "created_at"}, - PartKey: []string{"target"}, - SortKey: []string{"actor"}, + PartKey: []string{"target", "actor"}, } var followByTargetTable = table.New(followByTargetMetadata) @@ -161,17 +169,29 @@ type FollowByTarget struct { CreatedAt time.Time } +var blocksMetadata = table.Metadata{ + Name: "netsync.blocks", + Columns: []string{"actor", "rkey", "target", "created_at"}, + PartKey: []string{"actor", "rkey"}, +} +var blocksTable = table.New(blocksMetadata) + +type Block struct { + Actor string + Rkey string + Target string + CreatedAt time.Time +} + var blockByActorMetadata = table.Metadata{ Name: "netsync.blocks_by_actor", - Columns: []string{"actor", "rkey", "target", "created_at"}, - PartKey: []string{"actor"}, - SortKey: []string{"rkey"}, + Columns: []string{"actor", "target", "created_at"}, + PartKey: []string{"actor", "target"}, } var blockByActorTable = table.New(blockByActorMetadata) type BlockByActor struct { Actor string - Rkey string Target string CreatedAt time.Time } @@ -179,8 +199,7 @@ type BlockByActor struct { var blockByTargetMetadata = table.Metadata{ Name: "netsync.blocks_by_target", Columns: []string{"target", "actor", "created_at"}, - PartKey: []string{"target"}, - SortKey: []string{"actor"}, + PartKey: []string{"target", "actor"}, } var blockByTargetTable = table.New(blockByTargetMetadata) @@ -193,8 +212,7 @@ type BlockByTarget struct { var likesMetadata = table.Metadata{ Name: "netsync.likes", Columns: []string{"did", "rkey", "subject", "created_at"}, - PartKey: []string{"did"}, - SortKey: []string{"rkey"}, + PartKey: []string{"did", "rkey"}, } var likesTable = table.New(likesMetadata) @@ -208,8 +226,7 @@ type Like struct { var likeCountMetadata = table.Metadata{ Name: "netsync.like_counts", Columns: []string{"did", "nsid", "rkey", "count"}, - PartKey: []string{"did", "nsid"}, - SortKey: []string{"rkey"}, + PartKey: []string{"did", "nsid", "rkey"}, } var likeCountTable = table.New(likeCountMetadata) @@ -223,8 +240,7 @@ type LikeCount struct { var repostsMetadata = table.Metadata{ Name: "netsync.reposts", Columns: []string{"did", "rkey", "subject", "created_at"}, - PartKey: []string{"did"}, - SortKey: []string{"rkey"}, + PartKey: []string{"did", "rkey"}, } var repostsTable = table.New(repostsMetadata) @@ -238,8 +254,7 @@ type Repost struct { var repostCountMetadata = table.Metadata{ Name: "netsync.repost_counts", Columns: []string{"did", "nsid", "rkey", "count"}, - PartKey: []string{"did", "nsid"}, - SortKey: []string{"rkey"}, + PartKey: []string{"did", "nsid", "rkey"}, } var repostCountTable = table.New(repostCountMetadata) @@ -259,373 +274,57 @@ func (s *PlaybackState) SetupSchema() error { return fmt.Errorf("failed to create posts table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.post_windows (did text, window text, rkey text, created_at timestamp, PRIMARY KEY ((did, window), created_at));`); err != nil { - return fmt.Errorf("failed to create post windows table: %w", err) + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.posts_by_did (did text, rkey text, created_at timestamp, PRIMARY KEY (did, created_at)) WITH CLUSTERING ORDER BY (created_at DESC);`); err != nil { + return fmt.Errorf("failed to create posts by did table: %w", err) } if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.replies (parent_did text, parent_rkey text, child_did text, child_rkey text, created_at timestamp, PRIMARY KEY ((parent_did, parent_rkey), child_did, child_rkey));`); err != nil { return fmt.Errorf("failed to create replies table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY ((actor, rkey)));`); err != nil { + return fmt.Errorf("failed to create follows table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY ((actor, target)));`); err != nil { return fmt.Errorf("failed to create follows by actor table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_target (target text, actor text, created_at timestamp, PRIMARY KEY (target, actor));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.follows_by_target (target text, actor text, created_at timestamp, PRIMARY KEY ((target, actor)));`); err != nil { return fmt.Errorf("failed to create follows by target table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY (actor, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks (actor text, rkey text, target text, created_at timestamp, PRIMARY KEY ((actor, rkey)));`); err != nil { + return fmt.Errorf("failed to create blocks table: %w", err) + } + + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_actor (actor text, target text, created_at timestamp, PRIMARY KEY ((actor, target)));`); err != nil { return fmt.Errorf("failed to create blocks by actor table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_target (target text, actor text, created_at timestamp, PRIMARY KEY (target, actor));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.blocks_by_target (target text, actor text, created_at timestamp, PRIMARY KEY ((target, actor)));`); err != nil { return fmt.Errorf("failed to create blocks by target table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.likes (did text, rkey text, subject text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.likes (did text, rkey text, subject text, created_at timestamp, PRIMARY KEY ((did, rkey)));`); err != nil { return fmt.Errorf("failed to create likes table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.like_counts (did text, nsid text, rkey text, count counter, PRIMARY KEY ((did, nsid), rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.like_counts (did text, nsid text, rkey text, count counter, PRIMARY KEY ((did, nsid, rkey)));`); err != nil { return fmt.Errorf("failed to create like counts table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.reposts (did text, rkey text, subject text, created_at timestamp, PRIMARY KEY (did, rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.reposts (did text, rkey text, subject text, created_at timestamp, PRIMARY KEY ((did, rkey)));`); err != nil { return fmt.Errorf("failed to create reposts table: %w", err) } - if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.repost_counts (did text, nsid text, rkey text, count counter, PRIMARY KEY ((did, nsid), rkey));`); err != nil { + if err := s.ses.ExecStmt(`CREATE TABLE IF NOT EXISTS netsync.repost_counts (did text, nsid text, rkey text, count counter, PRIMARY KEY ((did, nsid, rkey)));`); err != nil { return fmt.Errorf("failed to create repost counts table: %w", err) } return nil } -func GetPostsForUser(cctx *cli.Context) error { - ctx := cctx.Context - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) - session, err := gocqlx.WrapSession(cluster.CreateSession()) - if err != nil { - return fmt.Errorf("failed to create scylla session: %w", err) - } - - args := cctx.Args() - if args.Len() != 1 { - return fmt.Errorf("must provide a did") - } - - did := args.First() - - limit := 500 - - numRuns := 2000 - maxConcurrent := 40 - sem := semaphore.NewWeighted(int64(maxConcurrent)) - - totalRowsRead := atomic.Uint64{} - - runtimes := make(chan time.Duration, numRuns) - - start := time.Now() - - // Compute window names - windowRangeStart := time.Date(2023, 8, 1, 0, 0, 0, 0, time.UTC) - windowRangeEnd := time.Now().UTC() - windowNames := []string{} - for windowRangeStart.Before(windowRangeEnd) { - windowNames = append(windowNames, windowRangeStart.Format("2006-01-02")) - windowRangeStart = windowRangeStart.Add(24 * time.Hour) - } - - // Run the query in numRuns goroutines - var pwg sync.WaitGroup - for i := 0; i < numRuns; i++ { - pwg.Add(1) - go func() error { - defer pwg.Done() - sem.Acquire(ctx, 1) - defer sem.Release(1) - iterStart := time.Now() - defer func() { - runtimes <- time.Since(iterStart) - }() - - // Query for all the posts in each window in parallel - var wg sync.WaitGroup - postWindows := []PostWindow{} - postWindowsLk := sync.Mutex{} - for _, windowName := range windowNames { - wg.Add(1) - go func(windowName string) { - defer wg.Done() - - postWindow := PostWindow{ - Did: did, - Window: windowName, - } - - windows := []PostWindow{} - - err = qb.Select(postWindowMetadata.Name). - Where(qb.Eq("did"), qb.Eq("window")). - OrderBy("created_at", qb.DESC).Limit(uint(limit)). - Query(session).BindStruct(&postWindow).SelectRelease(&windows) - if err != nil { - log.Errorf("failed to get post windows: %+v", err) - return - } - - postWindowsLk.Lock() - postWindows = append(postWindows, windows...) - postWindowsLk.Unlock() - }(windowName) - } - - wg.Wait() - - // log.Infow("got post windows", "count", len(postWindows), "duration", time.Since(iterStart)) - - // Sort the post windows by created at and limit - slices.SortFunc(postWindows, func(a, b PostWindow) int { - if a.CreatedAt.Before(b.CreatedAt) { - return -1 - } - if a.CreatedAt.After(b.CreatedAt) { - return 1 - } - return 0 - }) - postWindows = postWindows[:limit] - - totalRowsRead.Add(uint64(len(postWindows))) - - // Query for all the posts in each window in parallel - var wg2 sync.WaitGroup - postsLk := sync.Mutex{} - posts := []Post{} - for _, postWindow := range postWindows { - wg2.Add(1) - go func(postWindow PostWindow) { - defer wg2.Done() - post := Post{ - Did: postWindow.Did, - Rkey: postWindow.Rkey, - } - - err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) - if err != nil { - log.Errorf("failed to get post: %+v", err) - return - } - - postsLk.Lock() - posts = append(posts, post) - postsLk.Unlock() - }(postWindow) - } - wg2.Wait() - - totalRowsRead.Add(uint64(len(posts))) - - return nil - }() - } - - // Wait for all the queries to finish - pwg.Wait() - clockTime := time.Since(start) - close(runtimes) - - // Calculate the average runtime - var total time.Duration - for runtime := range runtimes { - total += runtime - } - avg := total / time.Duration(numRuns) - - p := message.NewPrinter(language.English) - - log.Info(p.Sprintf("got post page %d times (%d total reads) in %s (avg: %s, total: %s)", numRuns, totalRowsRead.Load(), clockTime, avg, total)) - - return nil -} - -func Query(cctx *cli.Context) error { - ctx := cctx.Context - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) - session, err := gocqlx.WrapSession(cluster.CreateSession()) - if err != nil { - return fmt.Errorf("failed to create scylla session: %w", err) - } - - args := cctx.Args() - if args.Len() != 1 { - return fmt.Errorf("must provide a post URI") - } - postURI := args.First() - - // at://did/app.bsky.feed.post/rkey - postURI = strings.TrimPrefix(postURI, "at://") - postParts := strings.Split(postURI, "/") - if len(postParts) != 3 { - return fmt.Errorf("invalid post URI: %s", postURI) - } - - numRuns := 500000 - maxConcurrent := 400 - sem := semaphore.NewWeighted(int64(maxConcurrent)) - - totalRowsRead := atomic.Uint64{} - - runtimes := make(chan time.Duration, numRuns) - - start := time.Now() - - // Run the query in numRuns goroutines - var pwg sync.WaitGroup - for i := 0; i < numRuns; i++ { - pwg.Add(1) - go func() error { - defer pwg.Done() - sem.Acquire(ctx, 1) - defer sem.Release(1) - iterStart := time.Now() - defer func() { - runtimes <- time.Since(iterStart) - }() - - // Get the post - post := Post{ - Did: postParts[0], - Rkey: postParts[2], - } - err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) - if err != nil { - return fmt.Errorf("failed to get post: %w", err) - } - - totalRowsRead.Add(1) - - // Get the replies - replyRefs := []Reply{} - err = repliesTable.SelectQuery(session).BindStruct(&Reply{ - ParentDid: postParts[0], - ParentRkey: postParts[2], - }).SelectRelease(&replyRefs) - if err != nil { - return fmt.Errorf("failed to get replies: %w", err) - } - - totalRowsRead.Add(uint64(len(replyRefs))) - - replies := []Post{} - lk := sync.Mutex{} - - // Resolve the replies as posts in parallel - var wg sync.WaitGroup - for i := range replyRefs { - wg.Add(1) - replyRef := replyRefs[i] - go func(replyRef Reply) { - defer wg.Done() - - reply := Post{ - Did: replyRef.ChildDid, - Rkey: replyRef.ChildRkey, - } - - err = postTable.GetQuery(session).BindStruct(&reply).GetRelease(&reply) - if err != nil { - log.Errorf("failed to get reply: %+v", err) - return - } - lk.Lock() - replies = append(replies, reply) - lk.Unlock() - }(replyRef) - } - - totalRowsRead.Add(uint64(len(replies))) - - // Resolve the parent up to the root - parents := []Post{} - if post.ParentDid != "" && post.ParentRkey != "" { - wg.Add(1) - go func() { - defer wg.Done() - parentDid := post.ParentDid - parentRkey := post.ParentRkey - for { - parent := Post{ - Did: parentDid, - Rkey: parentRkey, - } - err = postTable.GetQuery(session).BindStruct(&parent).GetRelease(&parent) - if err != nil && err != gocql.ErrNotFound { - log.Errorf("failed to get parent: %+v", err) - return - } - - parents = append(parents, parent) - - if parent.ParentDid == "" { - break - } - - parentDid = parent.ParentDid - parentRkey = parent.ParentRkey - } - }() - } - - totalRowsRead.Add(uint64(len(parents))) - - wg.Wait() - return nil - }() - } - - // // Print the thread - // p := message.NewPrinter(language.English) - // log.Debugf("post: %s", post.Content) - // log.Debugf("replies: %d", len(replies)) - // for _, reply := range replies { - // log.Debugf(" %s", reply.Content) - // } - // slices.Reverse(parents) - // log.Debugf("parents: %d", len(parents)) - // for _, parent := range parents { - // log.Debugf(" %s", parent.Content) - // } - - // log.Info(p.Sprintf("processed post with %d replies and resolved %d parents in %s", len(replies), len(parents), time.Since(start))) - - // Wait for all the queries to finish - pwg.Wait() - clockTime := time.Since(start) - close(runtimes) - - // Calculate the average runtime - var total time.Duration - for runtime := range runtimes { - total += runtime - } - avg := total / time.Duration(numRuns) - - p := message.NewPrinter(language.English) - - log.Info(p.Sprintf("processed post %d times (%d total reads) in %s (avg: %s, total: %s)", numRuns, totalRowsRead.Load(), clockTime, avg, total)) - - return nil -} - func Playback(cctx *cli.Context) error { ctx := cctx.Context ctx, cancel := context.WithCancel(ctx) @@ -803,19 +502,19 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return "", fmt.Errorf("failed to read repo from car: %w", err) } - maxBatchSize := 1000 + // maxBatchSize := 1000 - followByActorBatch := s.ses.NewBatch(gocql.LoggedBatch) - followByActorBatchSize := 0 + // followBatch := s.ses.NewBatch(gocql.LoggedBatch) + // followBatchSize := 0 - blockByActorBatch := s.ses.NewBatch(gocql.LoggedBatch) - blockByActorBatchSize := 0 + // blockBatch := s.ses.NewBatch(gocql.LoggedBatch) + // blockBatchSize := 0 - likeBatch := s.ses.NewBatch(gocql.LoggedBatch) - likeBatchSize := 0 + // likeBatch := s.ses.NewBatch(gocql.LoggedBatch) + // likeBatchSize := 0 - repostBatch := s.ses.NewBatch(gocql.LoggedBatch) - repostBatchSize := 0 + // repostBatch := s.ses.NewBatch(gocql.LoggedBatch) + // repostBatchSize := 0 err = r.ForEach(ctx, "", func(path string, _ cid.Cid) error { select { @@ -925,6 +624,7 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta post.SelfLabels = selfLabels } + // Insert into the DID+RKey lookup table insertPost := postTable.InsertQuery(s.ses) err = insertPost.BindStruct(&post).ExecRelease() if err != nil { @@ -932,20 +632,19 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } - // Insert into post windows using the day as the window - window := recCreatedAt.Format("2006-01-02") - insertPostWindow := postWindowTable.InsertQuery(s.ses) - err = insertPostWindow.BindStruct(&PostWindow{ + // Insert into post into author-indexed table + insertPostByDID := postsByDIDTable.InsertQuery(s.ses) + err = insertPostByDID.BindStruct(&PostByDID{ Did: did, - Window: window, Rkey: rkey, CreatedAt: recCreatedAt, }).ExecRelease() if err != nil { - log.Errorf("failed to exec post window: %w", err) + log.Errorf("failed to exec post by did: %w", err) return nil } + // Insert into the reply lookup table if len(parentParts) > 0 { insertReply := repliesTable.InsertQuery(s.ses) err = insertReply.BindStruct(&Reply{ @@ -968,20 +667,32 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } + // insertLike := likesTable.InsertQuery(s.ses) + // err = likeBatch.BindStruct(insertLike, &Like{ + // Did: did, + // Rkey: rkey, + // Subject: rec.Subject.Uri, + // CreatedAt: recCreatedAt, + // }) + // if err != nil { + // log.Errorf("failed to bind like: %w", err) + // return nil + // } + // likeBatchSize++ + + // Insert into the DID+RKey lookup table insertLike := likesTable.InsertQuery(s.ses) - err = likeBatch.BindStruct(insertLike, &Like{ + err = insertLike.BindStruct(&Like{ Did: did, Rkey: rkey, Subject: rec.Subject.Uri, CreatedAt: recCreatedAt, - }) + }).ExecRelease() if err != nil { - log.Errorf("failed to bind like: %w", err) + log.Errorf("failed to exec like: %w", err) return nil } - likeBatchSize++ - // Don't batch like count because the partition key isn't consistent subj := strings.TrimPrefix(rec.Subject.Uri, "at://") subjParts := strings.Split(subj, "/") if len(subjParts) != 3 { @@ -989,6 +700,7 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } + // Increment counter for subject updateLikeCount := likeCountTable.UpdateBuilder(). Add("count").Where(qb.Eq("did"), qb.Eq("nsid"), qb.Eq("rkey")).Query(s.ses). BindStruct(&LikeCount{ @@ -1012,20 +724,32 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } + // insertRepost := repostsTable.InsertQuery(s.ses) + // err = repostBatch.BindStruct(insertRepost, &Repost{ + // Did: did, + // Rkey: rkey, + // Subject: rec.Subject.Uri, + // CreatedAt: recCreatedAt, + // }) + // if err != nil { + // log.Errorf("failed to bind repost: %w", err) + // return nil + // } + // repostBatchSize++ + + // Insert into the DID+RKey lookup table insertRepost := repostsTable.InsertQuery(s.ses) - err = repostBatch.BindStruct(insertRepost, &Repost{ + err = insertRepost.BindStruct(&Repost{ Did: did, Rkey: rkey, Subject: rec.Subject.Uri, CreatedAt: recCreatedAt, - }) + }).ExecRelease() if err != nil { - log.Errorf("failed to bind repost: %w", err) + log.Errorf("failed to exec repost: %w", err) return nil } - repostBatchSize++ - // Don't batch repost count because the partition key isn't consistent subj := strings.TrimPrefix(rec.Subject.Uri, "at://") subjParts := strings.Split(subj, "/") if len(subjParts) != 3 { @@ -1033,6 +757,7 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } + // Increment counter for subject updateRepostCount := repostCountTable.UpdateBuilder(). Add("count").Where(qb.Eq("did"), qb.Eq("nsid"), qb.Eq("rkey")).Query(s.ses). BindStruct(&RepostCount{ @@ -1055,22 +780,46 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } - insertFollowByActor := followByActorTable.InsertQuery(s.ses) - insertFollowByTarget := followByTargetTable.InsertQuery(s.ses) - - err = followByActorBatch.BindStruct(insertFollowByActor, &FollowByActor{ + // Batch + // insertFollow := followsTable.InsertQuery(s.ses) + // err = followBatch.BindStruct(insertFollow, &FollowByActor{ + // Actor: did, + // Target: rec.Subject, + // CreatedAt: recCreatedAt, + // }) + // if err != nil { + // log.Errorf("failed to bind follow: %w", err) + // return nil + // } + // followBatchSize++ + + // Insert follow to DID+RKey Lookup table + insertFollow := followsTable.InsertQuery(s.ses) + err = insertFollow.BindStruct(&Follow{ Actor: did, Rkey: rkey, Target: rec.Subject, CreatedAt: recCreatedAt, - }) + }).ExecRelease() + if err != nil { + log.Errorf("failed to exec follow: %w", err) + return nil + } + + // Insert follow to Actor Lookup table + insertFollowByActor := followByActorTable.InsertQuery(s.ses) + err = insertFollowByActor.BindStruct(&FollowByActor{ + Actor: did, + Target: rec.Subject, + CreatedAt: recCreatedAt, + }).ExecRelease() if err != nil { - log.Errorf("failed to bind follow by actor: %w", err) + log.Errorf("failed to exec follow by actor: %w", err) return nil } - followByActorBatchSize++ - // Don't batch follow by target because the partition key isn't consistent + // Insert follow to Target Lookup table + insertFollowByTarget := followByTargetTable.InsertQuery(s.ses) err = insertFollowByTarget.BindStruct(&FollowByTarget{ Target: rec.Subject, Actor: did, @@ -1088,22 +837,33 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return nil } - insertBlockByActor := blockByActorTable.InsertQuery(s.ses) - insertBlockByTarget := blockByTargetTable.InsertQuery(s.ses) - - err = blockByActorBatch.BindStruct(insertBlockByActor, &BlockByActor{ + // Insert follow to DID+RKey Lookup table + insertBlock := blocksTable.InsertQuery(s.ses) + err = insertBlock.BindStruct(&Block{ Actor: did, Rkey: rkey, Target: rec.Subject, CreatedAt: recCreatedAt, - }) + }).ExecRelease() + if err != nil { + log.Errorf("failed to exec block: %w", err) + return nil + } + + // Insert block to Actor Lookup table + insertBlockByActor := blockByActorTable.InsertQuery(s.ses) + err = insertBlockByActor.BindStruct(&BlockByActor{ + Actor: did, + Target: rec.Subject, + CreatedAt: recCreatedAt, + }).ExecRelease() if err != nil { - log.Errorf("failed to bind block by actor: %w", err) + log.Errorf("failed to exec block by actor: %w", err) return nil } - blockByActorBatchSize++ - // Don't batch block by target because the partition key isn't consistent + // Insert block to Target Lookup table + insertBlockByTarget := blockByTargetTable.InsertQuery(s.ses) err = insertBlockByTarget.BindStruct(&BlockByTarget{ Target: rec.Subject, Actor: did, @@ -1119,41 +879,41 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta } } - if followByActorBatchSize >= maxBatchSize { - err = s.ses.ExecuteBatch(followByActorBatch) - if err != nil { - log.Errorf("failed to execute batch: %w", err) - } - followByActorBatch = s.ses.NewBatch(gocql.LoggedBatch) - followByActorBatchSize = 0 - } - - if blockByActorBatchSize >= maxBatchSize { - err = s.ses.ExecuteBatch(blockByActorBatch) - if err != nil { - log.Errorf("failed to execute batch: %w", err) - } - blockByActorBatch = s.ses.NewBatch(gocql.LoggedBatch) - blockByActorBatchSize = 0 - } - - if likeBatchSize >= maxBatchSize { - err = s.ses.ExecuteBatch(likeBatch) - if err != nil { - log.Errorf("failed to execute batch: %w", err) - } - likeBatch = s.ses.NewBatch(gocql.LoggedBatch) - likeBatchSize = 0 - } - - if repostBatchSize >= maxBatchSize { - err = s.ses.ExecuteBatch(repostBatch) - if err != nil { - log.Errorf("failed to execute batch: %w", err) - } - repostBatch = s.ses.NewBatch(gocql.LoggedBatch) - repostBatchSize = 0 - } + // if followBatchSize >= maxBatchSize { + // err = s.ses.ExecuteBatch(followBatch) + // if err != nil { + // log.Errorf("failed to execute batch: %w", err) + // } + // followBatch = s.ses.NewBatch(gocql.LoggedBatch) + // followBatchSize = 0 + // } + + // if blockBatchSize >= maxBatchSize { + // err = s.ses.ExecuteBatch(blockBatch) + // if err != nil { + // log.Errorf("failed to execute batch: %w", err) + // } + // blockBatch = s.ses.NewBatch(gocql.LoggedBatch) + // blockBatchSize = 0 + // } + + // if likeBatchSize >= maxBatchSize { + // err = s.ses.ExecuteBatch(likeBatch) + // if err != nil { + // log.Errorf("failed to execute batch: %w", err) + // } + // likeBatch = s.ses.NewBatch(gocql.LoggedBatch) + // likeBatchSize = 0 + // } + + // if repostBatchSize >= maxBatchSize { + // err = s.ses.ExecuteBatch(repostBatch) + // if err != nil { + // log.Errorf("failed to execute batch: %w", err) + // } + // repostBatch = s.ses.NewBatch(gocql.LoggedBatch) + // repostBatchSize = 0 + // } return nil }) @@ -1161,33 +921,33 @@ func (s *PlaybackState) processRepo(ctx context.Context, did string) (processSta return "failed (repo foreach)", fmt.Errorf("failed to process repo: %w", err) } - if followByActorBatchSize > 0 { - err = s.ses.ExecuteBatch(followByActorBatch) - if err != nil { - return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) - } - } + // if followBatchSize > 0 { + // err = s.ses.ExecuteBatch(followBatch) + // if err != nil { + // return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + // } + // } - if blockByActorBatchSize > 0 { - err = s.ses.ExecuteBatch(blockByActorBatch) - if err != nil { - return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) - } - } + // if blockBatchSize > 0 { + // err = s.ses.ExecuteBatch(blockBatch) + // if err != nil { + // return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + // } + // } - if likeBatchSize > 0 { - err = s.ses.ExecuteBatch(likeBatch) - if err != nil { - return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) - } - } + // if likeBatchSize > 0 { + // err = s.ses.ExecuteBatch(likeBatch) + // if err != nil { + // return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + // } + // } - if repostBatchSize > 0 { - err = s.ses.ExecuteBatch(repostBatch) - if err != nil { - return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) - } - } + // if repostBatchSize > 0 { + // err = s.ses.ExecuteBatch(repostBatch) + // if err != nil { + // return "failed (batch)", fmt.Errorf("failed to execute batch: %w", err) + // } + // } return "finished", nil } diff --git a/cmd/netsync/query.go b/cmd/netsync/query.go new file mode 100644 index 00000000..3045193c --- /dev/null +++ b/cmd/netsync/query.go @@ -0,0 +1,230 @@ +package main + +import ( + "context" + "fmt" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gocql/gocql" + "github.com/scylladb/gocqlx/v2" + "github.com/urfave/cli/v2" + "golang.org/x/sync/semaphore" + "golang.org/x/text/language" + "golang.org/x/text/message" +) + +func GetPostsForUser(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) + // session, err := gocqlx.WrapSession(cluster.CreateSession()) + // if err != nil { + // return fmt.Errorf("failed to create scylla session: %w", err) + // } + + // args := cctx.Args() + // if args.Len() != 1 { + // return fmt.Errorf("must provide a did") + // } + + // did := args.First() + + // limit := 500 + + // numRuns := 2000 + // maxConcurrent := 40 + // sem := semaphore.NewWeighted(int64(maxConcurrent)) + + // totalRowsRead := atomic.Uint64{} + + // runtimes := make(chan time.Duration, numRuns) + + // start := time.Now() + + return nil +} + +func Trim(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + return nil +} + +func Query(cctx *cli.Context) error { + ctx := cctx.Context + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) + session, err := gocqlx.WrapSession(cluster.CreateSession()) + if err != nil { + return fmt.Errorf("failed to create scylla session: %w", err) + } + + args := cctx.Args() + if args.Len() != 1 { + return fmt.Errorf("must provide a post URI") + } + postURI := args.First() + + // at://did/app.bsky.feed.post/rkey + postURI = strings.TrimPrefix(postURI, "at://") + postParts := strings.Split(postURI, "/") + if len(postParts) != 3 { + return fmt.Errorf("invalid post URI: %s", postURI) + } + + numRuns := 500000 + maxConcurrent := 400 + sem := semaphore.NewWeighted(int64(maxConcurrent)) + + totalRowsRead := atomic.Uint64{} + + runtimes := make(chan time.Duration, numRuns) + + start := time.Now() + + // Run the query in numRuns goroutines + var pwg sync.WaitGroup + for i := 0; i < numRuns; i++ { + pwg.Add(1) + go func() error { + defer pwg.Done() + sem.Acquire(ctx, 1) + defer sem.Release(1) + iterStart := time.Now() + defer func() { + runtimes <- time.Since(iterStart) + }() + + // Get the post + post := Post{ + Did: postParts[0], + Rkey: postParts[2], + } + err = postTable.GetQuery(session).BindStruct(&post).GetRelease(&post) + if err != nil { + return fmt.Errorf("failed to get post: %w", err) + } + + totalRowsRead.Add(1) + + // Get the replies + replyRefs := []Reply{} + err = repliesTable.SelectQuery(session).BindStruct(&Reply{ + ParentDid: postParts[0], + ParentRkey: postParts[2], + }).SelectRelease(&replyRefs) + if err != nil { + return fmt.Errorf("failed to get replies: %w", err) + } + + totalRowsRead.Add(uint64(len(replyRefs))) + + replies := []Post{} + lk := sync.Mutex{} + + // Resolve the replies as posts in parallel + var wg sync.WaitGroup + for i := range replyRefs { + wg.Add(1) + replyRef := replyRefs[i] + go func(replyRef Reply) { + defer wg.Done() + + reply := Post{ + Did: replyRef.ChildDid, + Rkey: replyRef.ChildRkey, + } + + err = postTable.GetQuery(session).BindStruct(&reply).GetRelease(&reply) + if err != nil { + log.Errorf("failed to get reply: %+v", err) + return + } + lk.Lock() + replies = append(replies, reply) + lk.Unlock() + }(replyRef) + } + + totalRowsRead.Add(uint64(len(replies))) + + // Resolve the parent up to the root + parents := []Post{} + if post.ParentDid != "" && post.ParentRkey != "" { + wg.Add(1) + go func() { + defer wg.Done() + parentDid := post.ParentDid + parentRkey := post.ParentRkey + for { + parent := Post{ + Did: parentDid, + Rkey: parentRkey, + } + err = postTable.GetQuery(session).BindStruct(&parent).GetRelease(&parent) + if err != nil && err != gocql.ErrNotFound { + log.Errorf("failed to get parent: %+v", err) + return + } + + parents = append(parents, parent) + + if parent.ParentDid == "" { + break + } + + parentDid = parent.ParentDid + parentRkey = parent.ParentRkey + } + }() + } + + totalRowsRead.Add(uint64(len(parents))) + + wg.Wait() + return nil + }() + } + + // // Print the thread + // p := message.NewPrinter(language.English) + // log.Debugf("post: %s", post.Content) + // log.Debugf("replies: %d", len(replies)) + // for _, reply := range replies { + // log.Debugf(" %s", reply.Content) + // } + // slices.Reverse(parents) + // log.Debugf("parents: %d", len(parents)) + // for _, parent := range parents { + // log.Debugf(" %s", parent.Content) + // } + + // log.Info(p.Sprintf("processed post with %d replies and resolved %d parents in %s", len(replies), len(parents), time.Since(start))) + + // Wait for all the queries to finish + pwg.Wait() + clockTime := time.Since(start) + close(runtimes) + + // Calculate the average runtime + var total time.Duration + for runtime := range runtimes { + total += runtime + } + avg := total / time.Duration(numRuns) + + p := message.NewPrinter(language.English) + + log.Info(p.Sprintf("processed post %d times (%d total reads) in %s (avg: %s, total: %s)", numRuns, totalRowsRead.Load(), clockTime, avg, total)) + + return nil +} -- 2.51.2 From f38ab76748b53853b4d60caebab7b31dac366b0d Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 7 Sep 2023 20:17:18 +0000 Subject: [PATCH 043/192] Add trim example --- Makefile | 2 +- cmd/netsync/query.go | 61 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1e762fb0..665d8864 100644 --- a/Makefile +++ b/Makefile @@ -118,7 +118,7 @@ sc-fire: # Fires supercollider .PHONY: run-netsync run-netsync: .env ## Runs netsync for local dev - go run ./cmd/netsync--checkout-limit 40 --worker-count 60 --out-dir ../netsync-out + go run ./cmd/netsync --checkout-limit 30 --worker-count 60 --out-dir ../netsync-out SCYLLA_VERSION := latest SCYLLA_CPU := 0 diff --git a/cmd/netsync/query.go b/cmd/netsync/query.go index 3045193c..00d8364a 100644 --- a/cmd/netsync/query.go +++ b/cmd/netsync/query.go @@ -10,6 +10,7 @@ import ( "github.com/gocql/gocql" "github.com/scylladb/gocqlx/v2" + "github.com/scylladb/gocqlx/v2/qb" "github.com/urfave/cli/v2" "golang.org/x/sync/semaphore" "golang.org/x/text/language" @@ -54,6 +55,66 @@ func Trim(cctx *cli.Context) error { ctx, cancel := context.WithCancel(ctx) defer cancel() + limit := uint(20_001) + + cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) + session, err := gocqlx.WrapSession(cluster.CreateSession()) + if err != nil { + return fmt.Errorf("failed to create scylla session: %w", err) + } + + args := cctx.Args() + if args.Len() != 1 { + return fmt.Errorf("must provide a did") + } + did := args.First() + + // Trim posts_by_did to 20,000 posts + // Select 20k posts, grab the `created_at` of the last one, then delete all posts with a `created_at` less than that + + start := time.Now() + + posts := []PostByDID{} + err = postsByDIDTable.SelectBuilder(). + Limit(limit). + OrderBy("created_at", qb.DESC). + QueryContext(ctx, session). + BindStruct(&PostByDID{Did: did}). + SelectRelease(&posts) + if err != nil { + return fmt.Errorf("failed to get posts: %w", err) + } + + if len(posts) == 0 { + return nil + } + + loadTime := time.Since(start) + + // Get the last post's created_at + lastPostCreatedAt := posts[len(posts)-2].CreatedAt + + log.Infow("got posts", "num_posts", len(posts), "last_post_created_at", lastPostCreatedAt, "duration", loadTime.String()) + + if len(posts) < int(limit) { + log.Info("no posts to trim") + return nil + } + + // Delete all posts with a created_at less than the last post's created_at + err = qb.Delete(postsByDIDTable.Name()). + Where(qb.Eq("did"), qb.Lt("created_at")). + QueryContext(ctx, session). + BindStruct(&PostByDID{Did: did, CreatedAt: lastPostCreatedAt}). + ExecRelease() + if err != nil { + return fmt.Errorf("failed to delete posts: %w", err) + } + + deleteTime := time.Since(start) + + log.Infow("deleted posts", "duration", deleteTime.String()) + return nil } -- 2.51.2 From 6b9b5d7b58d347691fe15760e8fb2da1e3faf77e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 7 Sep 2023 20:48:26 +0000 Subject: [PATCH 044/192] Trim all repos in parallel --- cmd/netsync/query.go | 143 ++++++++++++++++++++++++++++++++----------- 1 file changed, 106 insertions(+), 37 deletions(-) diff --git a/cmd/netsync/query.go b/cmd/netsync/query.go index 00d8364a..fd852f56 100644 --- a/cmd/netsync/query.go +++ b/cmd/netsync/query.go @@ -1,8 +1,10 @@ package main import ( + "bufio" "context" "fmt" + "os" "strings" "sync" "sync/atomic" @@ -57,63 +59,130 @@ func Trim(cctx *cli.Context) error { limit := uint(20_001) + // Read repo list + repoListFile, err := os.Open(cctx.String("repo-list")) + if err != nil { + return err + } + + fileScanner := bufio.NewScanner(repoListFile) + fileScanner.Split(bufio.ScanLines) + + repos := []string{} + + for fileScanner.Scan() { + repo := fileScanner.Text() + repos = append(repos, repo) + } + cluster := gocql.NewCluster(cctx.StringSlice("scylla-nodes")...) session, err := gocqlx.WrapSession(cluster.CreateSession()) if err != nil { return fmt.Errorf("failed to create scylla session: %w", err) } - args := cctx.Args() - if args.Len() != 1 { - return fmt.Errorf("must provide a did") - } - did := args.First() - // Trim posts_by_did to 20,000 posts // Select 20k posts, grab the `created_at` of the last one, then delete all posts with a `created_at` less than that - start := time.Now() - - posts := []PostByDID{} - err = postsByDIDTable.SelectBuilder(). - Limit(limit). - OrderBy("created_at", qb.DESC). - QueryContext(ctx, session). - BindStruct(&PostByDID{Did: did}). - SelectRelease(&posts) - if err != nil { - return fmt.Errorf("failed to get posts: %w", err) + type output struct { + err error + msg string + duration time.Duration } - if len(posts) == 0 { - return nil - } + st := time.Now() + outChan := make(chan output, len(repos)) + sem := semaphore.NewWeighted(int64(cctx.Int("worker-count"))) + var wg sync.WaitGroup - loadTime := time.Since(start) + // Run in parallel + for _, repo := range repos { + wg.Add(1) + go func(repo string) { + defer wg.Done() + sem.Acquire(ctx, 1) + defer sem.Release(1) - // Get the last post's created_at - lastPostCreatedAt := posts[len(posts)-2].CreatedAt + start := time.Now() + log := log.With("repo", repo) - log.Infow("got posts", "num_posts", len(posts), "last_post_created_at", lastPostCreatedAt, "duration", loadTime.String()) + posts := []PostByDID{} + err = postsByDIDTable.SelectBuilder(). + Limit(limit). + OrderBy("created_at", qb.DESC). + QueryContext(ctx, session). + BindStruct(&PostByDID{Did: repo}). + SelectRelease(&posts) + if err != nil { + outChan <- output{err: fmt.Errorf("failed to get posts: %w", err)} + return + } - if len(posts) < int(limit) { - log.Info("no posts to trim") - return nil + if len(posts) == 0 { + outChan <- output{msg: "no posts found for DID", duration: time.Since(start)} + return + } + + loadTime := time.Now() + + log.Debugw("got posts", "num_posts", len(posts), "duration", loadTime.Sub(start).String()) + + if len(posts) < int(limit) { + outChan <- output{msg: "no posts to trim", duration: time.Since(start)} + return + } + + // Get the last post's created_at + lastPostCreatedAt := posts[len(posts)-2].CreatedAt + + // Delete all posts with a created_at less than the last post's created_at + err = qb.Delete(postsByDIDTable.Name()). + Where(qb.Eq("did"), qb.Lt("created_at")). + QueryContext(ctx, session). + BindStruct(&PostByDID{Did: repo, CreatedAt: lastPostCreatedAt}). + ExecRelease() + if err != nil { + outChan <- output{err: fmt.Errorf("failed to delete posts: %w", err)} + return + } + + deleteTime := time.Now() + + log.Debugw("deleted posts", "duration", deleteTime.Sub(loadTime).String()) + + outChan <- output{msg: "trimmed posts", duration: time.Since(start)} + }(repo) } - // Delete all posts with a created_at less than the last post's created_at - err = qb.Delete(postsByDIDTable.Name()). - Where(qb.Eq("did"), qb.Lt("created_at")). - QueryContext(ctx, session). - BindStruct(&PostByDID{Did: did, CreatedAt: lastPostCreatedAt}). - ExecRelease() - if err != nil { - return fmt.Errorf("failed to delete posts: %w", err) + // Wait for all the queries to finish + wg.Wait() + close(outChan) + + end := time.Now() + + // Enumerate the results + var total time.Duration + errs := []error{} + successes := 0 + for out := range outChan { + if out.err != nil { + errs = append(errs, out.err) + } + if out.duration > 0 { + total += out.duration + successes++ + } + if out.msg != "" { + log.Debug(out.msg) + } } - deleteTime := time.Since(start) + // Calculate the average runtime + avg := total / time.Duration(successes) + + p := message.NewPrinter(language.English) - log.Infow("deleted posts", "duration", deleteTime.String()) + log.Info(p.Sprintf("trimmed %d repos (%d errors) in %s (avg: %s, total: %s)", successes, len(errs), end.Sub(st), avg, total)) return nil } -- 2.51.2 From e1dcb4f2f153f2bde7d42383c7e5ac16e03e4918 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 22:20:47 +0000 Subject: [PATCH 045/192] Add admin route to resync all repos for a PDS --- bgs/admin.go | 26 ++++++++++++++ bgs/bgs.go | 96 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/bgs/admin.go b/bgs/admin.go index 7452ffbe..3affe869 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -1,6 +1,7 @@ package bgs import ( + "context" "errors" "fmt" "net/http" @@ -430,3 +431,28 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { "success": "true", }) } + +func (bgs *BGS) handleAdminResyncPDS(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return fmt.Errorf("must pass a host") + } + + // Get the PDS from the DB + var pds models.PDS + if err := bgs.db.Where("host = ?", host).First(&pds).Error; err != nil { + return err + } + + go func() { + ctx := context.Background() + err := bgs.ResyncPDS(ctx, pds) + if err != nil { + log.Errorw("failed to resync PDS", "err", err, "pds", pds.Host) + } + }() + + return e.JSON(200, map[string]any{ + "message": "resync started...", + }) +} diff --git a/bgs/bgs.go b/bgs/bgs.go index bc2a9972..87a1dd77 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -28,6 +28,7 @@ import ( "github.com/bluesky-social/indigo/repomgr" "github.com/bluesky-social/indigo/util" "github.com/bluesky-social/indigo/xrpc" + "golang.org/x/time/rate" "github.com/gorilla/websocket" "github.com/ipfs/go-cid" @@ -311,6 +312,7 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { // PDS-related Admin API admin.GET("/pds/list", bgs.handleListPDSs) + admin.GET("/pds/resync", bgs.handleAdminResyncPDS) admin.POST("/pds/changeIngestRateLimit", bgs.handleAdminChangePDSRateLimit) admin.POST("/pds/changeCrawlRateLimit", bgs.handleAdminChangePDSCrawlLimit) admin.POST("/pds/block", bgs.handleBlockPDS) @@ -1099,3 +1101,97 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context) error { return nil } + +type repoHead struct { + Did string + Head string +} + +func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { + ctx, span := otel.Tracer("bgs").Start(ctx, "SyncPDS") + defer span.End() + + log := log.With("pds", pds.Host, "source", "sync_pds") + start := time.Now() + + log.Info("starting PDS resync") + + xrpcc := xrpc.Client{Host: pds.Host} + bgs.Index.ApplyPDSClientSettings(&xrpcc) + + limiter := rate.NewLimiter(rate.Limit(50), 1) + cursor := "" + limit := int64(500) + + repos := []repoHead{} + reposToCrawl := []*models.ActorInfo{} + + for { + if err := limiter.Wait(ctx); err != nil { + return fmt.Errorf("failed to wait for rate limiter: %w", err) + } + repoList, err := comatproto.SyncListRepos(ctx, &xrpcc, cursor, limit) + if err != nil { + return fmt.Errorf("failed to list repos: %w", err) + } + + for _, r := range repoList.Repos { + repos = append(repos, repoHead{ + Did: r.Did, + Head: r.Head, + }) + } + + if repoList.Cursor == nil || *repoList.Cursor != "" { + break + } + cursor = *repoList.Cursor + } + + repolistDone := time.Now() + + log.Infow("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) + + // Check the heads of repos against our own records + for _, r := range repos { + log := log.With("did", r.Did, "head", r.Head) + shouldCrawl := false + // Fetches the user if we have it, otherwise automatically enqueues it for crawling + ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) + if err != nil { + log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) + continue + } + + head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) + if err != nil { + log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) + shouldCrawl = true + } + + if head.String() != r.Head { + log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) + shouldCrawl = true + } + + if shouldCrawl { + reposToCrawl = append(reposToCrawl, ai) + } + } + + headCheckDone := time.Now() + + log.Infow("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) + + // Enqueue a crawl for each repo that needs it + for _, ai := range reposToCrawl { + err := bgs.Index.Crawler.Crawl(ctx, ai) + if err != nil { + log.Errorw("failed to enqueue crawl for repo during resync", "error", err, "uid", ai.Uid, "did", ai.Did) + } + } + + log.Infow("enqueued all crawls, exiting resync", "took", time.Now().Sub(start)) + + return nil +} -- 2.51.2 From 8493f153b4be1069a6f8e0f069843a349109e71d Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 22:26:48 +0000 Subject: [PATCH 046/192] Add protocol to xrpcc host --- bgs/bgs.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 87a1dd77..f1fcf05b 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1116,7 +1116,13 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { log.Info("starting PDS resync") - xrpcc := xrpc.Client{Host: pds.Host} + host := "http://" + if pds.SSL { + host = "https://" + } + host += pds.Host + + xrpcc := xrpc.Client{Host: host} bgs.Index.ApplyPDSClientSettings(&xrpcc) limiter := rate.NewLimiter(rate.Limit(50), 1) -- 2.51.2 From 3ad89e249acbe39d20871f7c97df9fcb3b1f88f9 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 22:54:16 +0000 Subject: [PATCH 047/192] More logging and some state tracking --- bgs/admin.go | 27 ++++++++++++- bgs/bgs.go | 107 +++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index 3affe869..3e8d92de 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -432,7 +432,7 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { }) } -func (bgs *BGS) handleAdminResyncPDS(e echo.Context) error { +func (bgs *BGS) handleAdminPostResyncPDS(e echo.Context) error { host := strings.TrimSpace(e.QueryParam("host")) if host == "" { return fmt.Errorf("must pass a host") @@ -456,3 +456,28 @@ func (bgs *BGS) handleAdminResyncPDS(e echo.Context) error { "message": "resync started...", }) } + +func (bgs *BGS) handleAdminGetResyncPDS(e echo.Context) error { + host := strings.TrimSpace(e.QueryParam("host")) + if host == "" { + return fmt.Errorf("must pass a host") + } + + // Get the PDS from the DB + var pds models.PDS + if err := bgs.db.Where("host = ?", host).First(&pds).Error; err != nil { + return err + } + + resync, found := bgs.GetResync(pds) + if !found { + return &echo.HTTPError{ + Code: 404, + Message: "no resync found for given PDS", + } + } + + return e.JSON(200, map[string]any{ + "resync": resync, + }) +} diff --git a/bgs/bgs.go b/bgs/bgs.go index f1fcf05b..0a325b93 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -78,6 +78,19 @@ type BGS struct { consumersLk sync.RWMutex nextConsumerID uint64 consumers map[uint64]*SocketConsumer + + // Management of Resyncs + pdsResyncsLk sync.RWMutex + pdsResyncs map[uint]*PDSResync +} + +type PDSResync struct { + PDS models.PDS `json:"pds"` + NumRepoPages int `json:"numRepoPages"` + NumRepos int `json:"numRepos"` + NumReposToResync int `json:"numReposToResync"` + Status string `json:"status"` + StatusChangedAt time.Time `json:"statusChangedAt"` } type SocketConsumer struct { @@ -106,6 +119,8 @@ func NewBGS(db *gorm.DB, ix *indexer.Indexer, repoman *repomgr.RepoManager, evtm consumersLk: sync.RWMutex{}, consumers: make(map[uint64]*SocketConsumer), + + pdsResyncs: make(map[uint]*PDSResync), } ix.CreateExternalUser = bgs.createExternalUser @@ -312,7 +327,8 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { // PDS-related Admin API admin.GET("/pds/list", bgs.handleListPDSs) - admin.GET("/pds/resync", bgs.handleAdminResyncPDS) + admin.POST("/pds/resync", bgs.handleAdminPostResyncPDS) + admin.GET("/pds/resync", bgs.handleAdminGetResyncPDS) admin.POST("/pds/changeIngestRateLimit", bgs.handleAdminChangePDSRateLimit) admin.POST("/pds/changeCrawlRateLimit", bgs.handleAdminChangePDSCrawlLimit) admin.POST("/pds/block", bgs.handleBlockPDS) @@ -1107,11 +1123,73 @@ type repoHead struct { Head string } +func (bgs *BGS) LoadOrStoreResync(pds models.PDS) (PDSResync, bool) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + if r, ok := bgs.pdsResyncs[pds.ID]; ok && r != nil { + return *r, false + } + + r := PDSResync{ + PDS: pds, + Status: "started", + StatusChangedAt: time.Now(), + } + + bgs.pdsResyncs[pds.ID] = &r + + return r, true +} + +func (bgs *BGS) GetResync(pds models.PDS) (PDSResync, bool) { + bgs.pdsResyncsLk.RLock() + defer bgs.pdsResyncsLk.RUnlock() + + if r, ok := bgs.pdsResyncs[pds.ID]; ok { + return *r, true + } + + return PDSResync{}, false +} + +func (bgs *BGS) UpdateResync(resync PDSResync) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + if r, ok := bgs.pdsResyncs[resync.PDS.ID]; ok { + *r = resync + } +} + +func (bgs *BGS) SetResyncStatus(id uint, status string) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + if r, ok := bgs.pdsResyncs[id]; ok { + r.Status = status + r.StatusChangedAt = time.Now() + } +} + +func (bgs *BGS) CompleteResync(resync PDSResync) { + bgs.pdsResyncsLk.Lock() + defer bgs.pdsResyncsLk.Unlock() + + if _, ok := bgs.pdsResyncs[resync.PDS.ID]; ok { + delete(bgs.pdsResyncs, resync.PDS.ID) + } +} + func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { ctx, span := otel.Tracer("bgs").Start(ctx, "SyncPDS") defer span.End() - log := log.With("pds", pds.Host, "source", "sync_pds") + resync, found := bgs.LoadOrStoreResync(pds) + if found { + return fmt.Errorf("resync already in progress") + } + start := time.Now() log.Info("starting PDS resync") @@ -1132,7 +1210,17 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { repos := []repoHead{} reposToCrawl := []*models.ActorInfo{} + pages := 0 + + bgs.SetResyncStatus(pds.ID, "listing repos") for { + pages++ + if pages%10 == 0 { + log.Infow("fetching PDS page during resync", "pages", pages, "total_repos", len(repos)) + resync.NumRepoPages = pages + resync.NumRepos = len(repos) + bgs.UpdateResync(resync) + } if err := limiter.Wait(ctx); err != nil { return fmt.Errorf("failed to wait for rate limiter: %w", err) } @@ -1154,13 +1242,22 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { cursor = *repoList.Cursor } + resync.NumRepoPages = pages + resync.NumRepos = len(repos) + bgs.UpdateResync(resync) + repolistDone := time.Now() log.Infow("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) + bgs.SetResyncStatus(pds.ID, "checking heads") // Check the heads of repos against our own records - for _, r := range repos { + for i, r := range repos { log := log.With("did", r.Did, "head", r.Head) + if i%10_000 == 0 { + log.Infow("checking repo head during resync", "repos_checked", i, "total_repos", len(repos)) + } + shouldCrawl := false // Fetches the user if we have it, otherwise automatically enqueues it for crawling ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) @@ -1181,6 +1278,8 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } if shouldCrawl { + resync.NumReposToResync++ + bgs.UpdateResync(resync) reposToCrawl = append(reposToCrawl, ai) } } @@ -1189,6 +1288,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { log.Infow("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) + bgs.SetResyncStatus(pds.ID, "crawling") // Enqueue a crawl for each repo that needs it for _, ai := range reposToCrawl { err := bgs.Index.Crawler.Crawl(ctx, ai) @@ -1198,6 +1298,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } log.Infow("enqueued all crawls, exiting resync", "took", time.Now().Sub(start)) + bgs.CompleteResync(resync) return nil } -- 2.51.2 From ccb291457972f082954861ac29d35c772f6ce0a1 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 23:10:40 +0000 Subject: [PATCH 048/192] Logging levels --- bgs/bgs.go | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 0a325b93..56ce39d6 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1157,9 +1157,7 @@ func (bgs *BGS) UpdateResync(resync PDSResync) { bgs.pdsResyncsLk.Lock() defer bgs.pdsResyncsLk.Unlock() - if r, ok := bgs.pdsResyncs[resync.PDS.ID]; ok { - *r = resync - } + bgs.pdsResyncs[resync.PDS.ID] = &resync } func (bgs *BGS) SetResyncStatus(id uint, status string) { @@ -1176,9 +1174,7 @@ func (bgs *BGS) CompleteResync(resync PDSResync) { bgs.pdsResyncsLk.Lock() defer bgs.pdsResyncsLk.Unlock() - if _, ok := bgs.pdsResyncs[resync.PDS.ID]; ok { - delete(bgs.pdsResyncs, resync.PDS.ID) - } + delete(bgs.pdsResyncs, resync.PDS.ID) } func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { @@ -1192,7 +1188,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { start := time.Now() - log.Info("starting PDS resync") + log.Warn("starting PDS resync") host := "http://" if pds.SSL { @@ -1216,7 +1212,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { for { pages++ if pages%10 == 0 { - log.Infow("fetching PDS page during resync", "pages", pages, "total_repos", len(repos)) + log.Warnw("fetching PDS page during resync", "pages", pages, "total_repos", len(repos)) resync.NumRepoPages = pages resync.NumRepos = len(repos) bgs.UpdateResync(resync) @@ -1248,14 +1244,14 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { repolistDone := time.Now() - log.Infow("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) + log.Warnw("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) bgs.SetResyncStatus(pds.ID, "checking heads") // Check the heads of repos against our own records for i, r := range repos { log := log.With("did", r.Did, "head", r.Head) if i%10_000 == 0 { - log.Infow("checking repo head during resync", "repos_checked", i, "total_repos", len(repos)) + log.Warnw("checking repo head during resync", "repos_checked", i, "total_repos", len(repos)) } shouldCrawl := false @@ -1286,7 +1282,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { headCheckDone := time.Now() - log.Infow("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) + log.Warnw("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) bgs.SetResyncStatus(pds.ID, "crawling") // Enqueue a crawl for each repo that needs it @@ -1297,7 +1293,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } } - log.Infow("enqueued all crawls, exiting resync", "took", time.Now().Sub(start)) + log.Warnw("enqueued all crawls, exiting resync", "took", time.Now().Sub(start)) bgs.CompleteResync(resync) return nil -- 2.51.2 From ca47046bc2c8cf3c3dcc6fe1d59555853a326c5e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 23:22:10 +0000 Subject: [PATCH 049/192] Debugging --- bgs/bgs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 56ce39d6..059c891b 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1233,6 +1233,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } if repoList.Cursor == nil || *repoList.Cursor != "" { + log.Warnw("no more repos to list", "cursor", cursor) break } cursor = *repoList.Cursor -- 2.51.2 From 22cd3963d29c79731fda38b874ebe6be437027a1 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 23:24:40 +0000 Subject: [PATCH 050/192] I'm dumb --- bgs/bgs.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 059c891b..44534df8 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1232,8 +1232,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { }) } - if repoList.Cursor == nil || *repoList.Cursor != "" { - log.Warnw("no more repos to list", "cursor", cursor) + if repoList.Cursor == nil || *repoList.Cursor == "" { break } cursor = *repoList.Cursor -- 2.51.2 From 207e31393b44de7fe0c31944760b91abbedc48da Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 26 Sep 2023 23:48:45 +0000 Subject: [PATCH 051/192] Show repo head checking progress --- bgs/bgs.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 44534df8..1bc0d186 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -88,6 +88,7 @@ type PDSResync struct { PDS models.PDS `json:"pds"` NumRepoPages int `json:"numRepoPages"` NumRepos int `json:"numRepos"` + NumReposChecked int `json:"numReposChecked"` NumReposToResync int `json:"numReposToResync"` Status string `json:"status"` StatusChangedAt time.Time `json:"statusChangedAt"` @@ -1252,6 +1253,8 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { log := log.With("did", r.Did, "head", r.Head) if i%10_000 == 0 { log.Warnw("checking repo head during resync", "repos_checked", i, "total_repos", len(repos)) + resync.NumReposChecked = i + bgs.UpdateResync(resync) } shouldCrawl := false -- 2.51.2 From 7c4e1ebb9e9d507cd92ee1d8555c93d9d0d3a7d4 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 00:50:55 +0000 Subject: [PATCH 052/192] Parallelize repo head checks --- bgs/bgs.go | 92 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 63 insertions(+), 29 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 1bc0d186..e9986f69 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -28,6 +28,7 @@ import ( "github.com/bluesky-social/indigo/repomgr" "github.com/bluesky-social/indigo/util" "github.com/bluesky-social/indigo/xrpc" + "golang.org/x/sync/semaphore" "golang.org/x/time/rate" "github.com/gorilla/websocket" @@ -1205,7 +1206,6 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { limit := int64(500) repos := []repoHead{} - reposToCrawl := []*models.ActorInfo{} pages := 0 @@ -1244,45 +1244,79 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { bgs.UpdateResync(resync) repolistDone := time.Now() + wg := sync.WaitGroup{} + sem := semaphore.NewWeighted(40) + + numReposChecked := 0 + numReposToResync := 0 + checkLk := sync.Mutex{} + + rtcLk := sync.Mutex{} + reposToCrawl := []*models.ActorInfo{} log.Warnw("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) bgs.SetResyncStatus(pds.ID, "checking heads") // Check the heads of repos against our own records - for i, r := range repos { - log := log.With("did", r.Did, "head", r.Head) - if i%10_000 == 0 { - log.Warnw("checking repo head during resync", "repos_checked", i, "total_repos", len(repos)) - resync.NumReposChecked = i - bgs.UpdateResync(resync) - } + for _, r := range repos { + wg.Add(1) + go func(r repoHead) { + defer wg.Done() + defer sem.Release(1) + if err := sem.Acquire(ctx, 1); err != nil { + log.Errorw("failed to acquire semaphore", "error", err) + return + } - shouldCrawl := false - // Fetches the user if we have it, otherwise automatically enqueues it for crawling - ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) - if err != nil { - log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) - continue - } + log := log.With("did", r.Did, "head", r.Head) - head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) - if err != nil { - log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) - shouldCrawl = true - } + checkLk.Lock() + numReposChecked++ + nc := numReposChecked + checkLk.Unlock() - if head.String() != r.Head { - log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) - shouldCrawl = true - } + if nc%10_000 == 0 { + log.Warnw("checking repo head during resync", "repos_checked", nc, "total_repos", len(repos)) + resync.NumReposChecked = nc + bgs.UpdateResync(resync) + } - if shouldCrawl { - resync.NumReposToResync++ - bgs.UpdateResync(resync) - reposToCrawl = append(reposToCrawl, ai) - } + shouldCrawl := false + // Fetches the user if we have it, otherwise automatically enqueues it for crawling + ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) + if err != nil { + log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) + return + } + + head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) + if err != nil { + log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) + shouldCrawl = true + } + + if head.String() != r.Head { + log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) + shouldCrawl = true + } + + if shouldCrawl { + checkLk.Lock() + numReposToResync++ + resync.NumReposToResync = numReposToResync + checkLk.Unlock() + + bgs.UpdateResync(resync) + + rtcLk.Lock() + reposToCrawl = append(reposToCrawl, ai) + rtcLk.Unlock() + } + }(r) } + wg.Wait() + headCheckDone := time.Now() log.Warnw("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) -- 2.51.2 From 912dd5200bfa09edacf0200589335cc65d08184b Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 00:52:50 +0000 Subject: [PATCH 053/192] Fix status --- bgs/bgs.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index e9986f69..f233bbd7 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1162,7 +1162,7 @@ func (bgs *BGS) UpdateResync(resync PDSResync) { bgs.pdsResyncs[resync.PDS.ID] = &resync } -func (bgs *BGS) SetResyncStatus(id uint, status string) { +func (bgs *BGS) SetResyncStatus(id uint, status string) PDSResync { bgs.pdsResyncsLk.Lock() defer bgs.pdsResyncsLk.Unlock() @@ -1170,6 +1170,8 @@ func (bgs *BGS) SetResyncStatus(id uint, status string) { r.Status = status r.StatusChangedAt = time.Now() } + + return *bgs.pdsResyncs[id] } func (bgs *BGS) CompleteResync(resync PDSResync) { @@ -1209,7 +1211,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { pages := 0 - bgs.SetResyncStatus(pds.ID, "listing repos") + resync = bgs.SetResyncStatus(pds.ID, "listing repos") for { pages++ if pages%10 == 0 { @@ -1256,7 +1258,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { log.Warnw("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) - bgs.SetResyncStatus(pds.ID, "checking heads") + resync = bgs.SetResyncStatus(pds.ID, "checking heads") // Check the heads of repos against our own records for _, r := range repos { wg.Add(1) @@ -1321,7 +1323,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { log.Warnw("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) - bgs.SetResyncStatus(pds.ID, "crawling") + resync = bgs.SetResyncStatus(pds.ID, "crawling") // Enqueue a crawl for each repo that needs it for _, ai := range reposToCrawl { err := bgs.Index.Crawler.Crawl(ctx, ai) -- 2.51.2 From 3d8fc90df28b9759ad904e3197e88c85b2f2454e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:06:33 +0000 Subject: [PATCH 054/192] Use channels --- bgs/bgs.go | 116 ++++++++++++++++++++++++++--------------------------- 1 file changed, 57 insertions(+), 59 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index f233bbd7..338b5f6a 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1125,6 +1125,11 @@ type repoHead struct { Head string } +type headCheckResult struct { + ai *models.ActorInfo + err error +} + func (bgs *BGS) LoadOrStoreResync(pds models.PDS) (PDSResync, bool) { bgs.pdsResyncsLk.Lock() defer bgs.pdsResyncsLk.Unlock() @@ -1246,78 +1251,71 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { bgs.UpdateResync(resync) repolistDone := time.Now() - wg := sync.WaitGroup{} - sem := semaphore.NewWeighted(40) - - numReposChecked := 0 - numReposToResync := 0 - checkLk := sync.Mutex{} - - rtcLk := sync.Mutex{} - reposToCrawl := []*models.ActorInfo{} log.Warnw("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) - resync = bgs.SetResyncStatus(pds.ID, "checking heads") - // Check the heads of repos against our own records - for _, r := range repos { - wg.Add(1) - go func(r repoHead) { - defer wg.Done() - defer sem.Release(1) - if err := sem.Acquire(ctx, 1); err != nil { - log.Errorw("failed to acquire semaphore", "error", err) - return - } - log := log.With("did", r.Did, "head", r.Head) + // Create a buffered channel for collecting results + results := make(chan headCheckResult, len(repos)) - checkLk.Lock() - numReposChecked++ - nc := numReposChecked - checkLk.Unlock() - - if nc%10_000 == 0 { - log.Warnw("checking repo head during resync", "repos_checked", nc, "total_repos", len(repos)) - resync.NumReposChecked = nc - bgs.UpdateResync(resync) - } - - shouldCrawl := false - // Fetches the user if we have it, otherwise automatically enqueues it for crawling - ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) - if err != nil { - log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) - return - } + sem := semaphore.NewWeighted(40) + // Create a worker function for goroutines + worker := func(r repoHead) { + if err := sem.Acquire(ctx, 1); err != nil { + log.Errorw("failed to acquire semaphore", "error", err) + return + } + defer sem.Release(1) - head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) - if err != nil { - log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) - shouldCrawl = true - } + log := log.With("did", r.Did, "head", r.Head) + // Fetches the user if we have it, otherwise automatically enqueues it for crawling + ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) + if err != nil { + log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) + results <- headCheckResult{err: err} + } - if head.String() != r.Head { - log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) - shouldCrawl = true - } + head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) + if err != nil { + log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) + return + } - if shouldCrawl { - checkLk.Lock() - numReposToResync++ - resync.NumReposToResync = numReposToResync - checkLk.Unlock() + if head.String() != r.Head { + log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) + results <- headCheckResult{ai: ai} + return + } - bgs.UpdateResync(resync) + results <- headCheckResult{} + } - rtcLk.Lock() - reposToCrawl = append(reposToCrawl, ai) - rtcLk.Unlock() - } + for _, r := range repos { + go func(r repoHead) { + worker(r) }(r) } - wg.Wait() + var reposToCrawl []*models.ActorInfo + for i := 0; i < len(repos); i++ { + res := <-results + if res.err != nil { + log.Errorw("failed to process repo during resync", "error", res.err) + } + if res.ai != nil { + reposToCrawl = append(reposToCrawl, res.ai) + } + if i%10_000 == 0 { + log.Warnw("checked heads during resync", "num_repos_checked", i, "num_repos_to_crawl", len(reposToCrawl), "took", time.Now().Sub(resync.StatusChangedAt)) + resync.NumReposChecked = i + resync.NumReposToResync = len(reposToCrawl) + bgs.UpdateResync(resync) + } + } + + resync.NumReposChecked = len(repos) + resync.NumReposToResync = len(reposToCrawl) + bgs.UpdateResync(resync) headCheckDone := time.Now() -- 2.51.2 From 7e44f54c546d0e1da9eaa401ccc1cdc758e8394e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:09:21 +0000 Subject: [PATCH 055/192] Cralw as we go --- bgs/bgs.go | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 338b5f6a..595f955f 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1296,41 +1296,33 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { }(r) } - var reposToCrawl []*models.ActorInfo + var numReposToResync int for i := 0; i < len(repos); i++ { res := <-results if res.err != nil { log.Errorw("failed to process repo during resync", "error", res.err) + } if res.ai != nil { - reposToCrawl = append(reposToCrawl, res.ai) + numReposToResync++ + err := bgs.Index.Crawler.Crawl(ctx, res.ai) + if err != nil { + log.Errorw("failed to enqueue crawl for repo during resync", "error", err, "uid", res.ai.Uid, "did", res.ai.Did) + } } if i%10_000 == 0 { - log.Warnw("checked heads during resync", "num_repos_checked", i, "num_repos_to_crawl", len(reposToCrawl), "took", time.Now().Sub(resync.StatusChangedAt)) + log.Warnw("checked heads during resync", "num_repos_checked", i, "num_repos_to_crawl", numReposToResync, "took", time.Now().Sub(resync.StatusChangedAt)) resync.NumReposChecked = i - resync.NumReposToResync = len(reposToCrawl) + resync.NumReposToResync = numReposToResync bgs.UpdateResync(resync) } } resync.NumReposChecked = len(repos) - resync.NumReposToResync = len(reposToCrawl) + resync.NumReposToResync = numReposToResync bgs.UpdateResync(resync) - headCheckDone := time.Now() - - log.Warnw("checked all heads, starting crawls", "num_repos_to_crawl", len(reposToCrawl), "took", headCheckDone.Sub(repolistDone)) - - resync = bgs.SetResyncStatus(pds.ID, "crawling") - // Enqueue a crawl for each repo that needs it - for _, ai := range reposToCrawl { - err := bgs.Index.Crawler.Crawl(ctx, ai) - if err != nil { - log.Errorw("failed to enqueue crawl for repo during resync", "error", err, "uid", ai.Uid, "did", ai.Did) - } - } - - log.Warnw("enqueued all crawls, exiting resync", "took", time.Now().Sub(start)) + log.Warnw("enqueued all crawls, exiting resync", "took", time.Now().Sub(start), "num_repos_to_crawl", numReposToResync) bgs.CompleteResync(resync) return nil -- 2.51.2 From e2fadcd17e3e21d111cbecc01a0c15e614719cf1 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:10:16 +0000 Subject: [PATCH 056/192] Cleanup --- bgs/bgs.go | 56 +++++++++++++++++++++++++----------------------------- 1 file changed, 26 insertions(+), 30 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 595f955f..8eff8d72 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1257,42 +1257,38 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { // Create a buffered channel for collecting results results := make(chan headCheckResult, len(repos)) - sem := semaphore.NewWeighted(40) - // Create a worker function for goroutines - worker := func(r repoHead) { - if err := sem.Acquire(ctx, 1); err != nil { - log.Errorw("failed to acquire semaphore", "error", err) - return - } - defer sem.Release(1) - log := log.With("did", r.Did, "head", r.Head) - // Fetches the user if we have it, otherwise automatically enqueues it for crawling - ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) - if err != nil { - log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) - results <- headCheckResult{err: err} - } + // Check repo heads against our local copy and enqueue crawls for any that are out of date + for _, r := range repos { + go func(r repoHead) { + if err := sem.Acquire(ctx, 1); err != nil { + log.Errorw("failed to acquire semaphore", "error", err) + return + } + defer sem.Release(1) - head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) - if err != nil { - log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) - return - } + log := log.With("did", r.Did, "head", r.Head) + // Fetches the user if we have it, otherwise automatically enqueues it for crawling + ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) + if err != nil { + log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) + results <- headCheckResult{err: err} + } - if head.String() != r.Head { - log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) - results <- headCheckResult{ai: ai} - return - } + head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) + if err != nil { + log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) + return + } - results <- headCheckResult{} - } + if head.String() != r.Head { + log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) + results <- headCheckResult{ai: ai} + return + } - for _, r := range repos { - go func(r repoHead) { - worker(r) + results <- headCheckResult{} }(r) } -- 2.51.2 From 8fe1d86cee3c6be0e1e763d4ef4778a34ba73ca8 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:10:38 +0000 Subject: [PATCH 057/192] Fix logs --- bgs/bgs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 8eff8d72..36db6bc3 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1187,9 +1187,9 @@ func (bgs *BGS) CompleteResync(resync PDSResync) { } func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { - ctx, span := otel.Tracer("bgs").Start(ctx, "SyncPDS") + ctx, span := otel.Tracer("bgs").Start(ctx, "ResyncPDS") defer span.End() - log := log.With("pds", pds.Host, "source", "sync_pds") + log := log.With("pds", pds.Host, "source", "resync_pds") resync, found := bgs.LoadOrStoreResync(pds) if found { return fmt.Errorf("resync already in progress") -- 2.51.2 From 6e499f12df8cfb6ae4dde5d16004e17102d2c422 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:16:21 +0000 Subject: [PATCH 058/192] Had a thing flipped --- bgs/bgs.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 36db6bc3..378ea95f 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1135,7 +1135,7 @@ func (bgs *BGS) LoadOrStoreResync(pds models.PDS) (PDSResync, bool) { defer bgs.pdsResyncsLk.Unlock() if r, ok := bgs.pdsResyncs[pds.ID]; ok && r != nil { - return *r, false + return *r, true } r := PDSResync{ @@ -1146,7 +1146,7 @@ func (bgs *BGS) LoadOrStoreResync(pds models.PDS) (PDSResync, bool) { bgs.pdsResyncs[pds.ID] = &r - return r, true + return r, false } func (bgs *BGS) GetResync(pds models.PDS) (PDSResync, bool) { -- 2.51.2 From a6071d467471ccf03a4443d25105f13f84ffa6cf Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:17:52 +0000 Subject: [PATCH 059/192] More logging --- bgs/bgs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 378ea95f..05fb5cd9 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1226,10 +1226,12 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { bgs.UpdateResync(resync) } if err := limiter.Wait(ctx); err != nil { + log.Errorw("failed to wait for rate limiter", "error", err) return fmt.Errorf("failed to wait for rate limiter: %w", err) } repoList, err := comatproto.SyncListRepos(ctx, &xrpcc, cursor, limit) if err != nil { + log.Errorw("failed to list repos", "error", err) return fmt.Errorf("failed to list repos: %w", err) } -- 2.51.2 From d871fcf1270d7d649a6d6028e99f86f69aa7a989 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:27:36 +0000 Subject: [PATCH 060/192] Bugfix --- bgs/bgs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 05fb5cd9..05b8ba90 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1276,6 +1276,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { if err != nil { log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) results <- headCheckResult{err: err} + return } head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) -- 2.51.2 From c3cef8acaa9b6b582581869d4ba2333ff94fe791 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:32:36 +0000 Subject: [PATCH 061/192] Another bug --- bgs/bgs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 05b8ba90..1c8f3221 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1282,6 +1282,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) if err != nil { log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) + results <- headCheckResult{ai: ai} return } -- 2.51.2 From 81cace582efdbc7efb35723febe6268d08772715 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:33:15 +0000 Subject: [PATCH 062/192] Always emit a result for good measure --- bgs/bgs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 1c8f3221..56cbaa1a 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1266,6 +1266,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { go func(r repoHead) { if err := sem.Acquire(ctx, 1); err != nil { log.Errorw("failed to acquire semaphore", "error", err) + results <- headCheckResult{err: err} return } defer sem.Release(1) -- 2.51.2 From 75b93bdda4c6b501341b4a1be2d6eb16dd0f00a4 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:38:39 +0000 Subject: [PATCH 063/192] Cleanup resync --- bgs/bgs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 56cbaa1a..2a2464a0 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1194,6 +1194,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { if found { return fmt.Errorf("resync already in progress") } + defer bgs.CompleteResync(resync) start := time.Now() @@ -1324,7 +1325,6 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { bgs.UpdateResync(resync) log.Warnw("enqueued all crawls, exiting resync", "took", time.Now().Sub(start), "num_repos_to_crawl", numReposToResync) - bgs.CompleteResync(resync) return nil } -- 2.51.2 From e16460c015c88fb0b23c03a554a9f880dce3d8cc Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 01:59:57 +0000 Subject: [PATCH 064/192] Track crawls in the right place --- indexer/crawler.go | 2 ++ indexer/indexer.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/indexer/crawler.go b/indexer/crawler.go index 0138cfbf..c5ed5e55 100644 --- a/indexer/crawler.go +++ b/indexer/crawler.go @@ -219,6 +219,8 @@ func (c *CrawlDispatcher) Crawl(ctx context.Context, ai *models.ActorInfo) error panic("must have pds for user in queue") } + userCrawlsEnqueued.Inc() + ctx, span := otel.Tracer("crawler").Start(ctx, "addToCrawler") defer span.End() diff --git a/indexer/indexer.go b/indexer/indexer.go index 711b6d18..bd44a3bc 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -317,8 +317,6 @@ func (ix *Indexer) createMissingUserRecord(ctx context.Context, did string) (*mo } func (ix *Indexer) addUserToCrawler(ctx context.Context, ai *models.ActorInfo) error { - userCrawlsEnqueued.Inc() - log.Infow("Sending user to crawler: ", "did", ai.Did) if ix.Crawler == nil { return nil -- 2.51.2 From 5a4f71b2536c8290cabe860f525a58cacd8d0c90 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 26 Sep 2023 19:17:55 -0700 Subject: [PATCH 065/192] palomar: default q to '*' when there is a filter --- search/parse_query.go | 3 +++ search/parse_query_test.go | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/search/parse_query.go b/search/parse_query.go index c08c0414..24d20552 100644 --- a/search/parse_query.go +++ b/search/parse_query.go @@ -66,5 +66,8 @@ func ParseQuery(ctx context.Context, dir identity.Directory, raw string) (string } } } + if out == "" && len(filters) >= 1 { + out = "*" + } return out, filters } diff --git a/search/parse_query_test.go b/search/parse_query_test.go index 18a00bf5..563139db 100644 --- a/search/parse_query_test.go +++ b/search/parse_query_test.go @@ -40,4 +40,9 @@ func TestParseQuery(t *testing.T) { q, f = ParseQuery(ctx, &dir, p3) assert.Equal("known", q) assert.Equal(1, len(f)) + + p4 := "from:known.example.com" + q, f = ParseQuery(ctx, &dir, p4) + assert.Equal("*", q) + assert.Equal(1, len(f)) } -- 2.51.2 From f554233358e8b527f14e6dfb6bb127aecde01d89 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 27 Sep 2023 10:11:13 -0700 Subject: [PATCH 066/192] add some extra logs for debugging sync issues --- bgs/bgs.go | 12 ++++++++---- indexer/indexer.go | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index bc2a9972..8d92f9a1 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -744,8 +744,12 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event return nil } + if evt.Rebase { + return fmt.Errorf("rebase was true in event seq:%d,host:%s", evt.Seq, host.Host) + } + // skip the fast path for rebases or if the user is already in the slow path - if evt.Rebase || bgs.Index.Crawler.RepoInSlowPath(ctx, host, u.ID) { + if bgs.Index.Crawler.RepoInSlowPath(ctx, host, u.ID) { rebasesCounter.WithLabelValues(host.Host).Add(1) ai, err := bgs.Index.LookupUser(ctx, u.ID) if err != nil { @@ -767,9 +771,9 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event log.Warnw("failed handling event", "err", err, "host", host.Host, "seq", evt.Seq, "repo", u.Did, "prev", stringLink(evt.Prev), "commit", evt.Commit.String()) if errors.Is(err, carstore.ErrRepoBaseMismatch) || ipld.IsNotFound(err) { - ai, err := bgs.Index.LookupUser(ctx, u.ID) - if err != nil { - return fmt.Errorf("failed to look up user (err case): %w", err) + ai, lerr := bgs.Index.LookupUser(ctx, u.ID) + if lerr != nil { + return fmt.Errorf("failed to look up user %s (%d) (err case: %s): %w", u.Did, u.ID, err, lerr) } span.SetAttributes(attribute.Bool("catchup_queue", true)) diff --git a/indexer/indexer.go b/indexer/indexer.go index 711b6d18..95a258ca 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -454,7 +454,7 @@ func (ix *Indexer) FetchAndIndexRepo(ctx context.Context, job *crawlWork) error for i, j := range job.catchup { catchupEventsProcessed.Inc() if err := ix.repomgr.HandleExternalUserEvent(ctx, pds.ID, ai.Uid, ai.Did, j.evt.Since, j.evt.Rev, j.evt.Blocks, j.evt.Ops); err != nil { - log.Errorw("buffered event catchup failed", "error", err, "did", ai.Did, "i", i, "jobCount", len(job.catchup)) + log.Errorw("buffered event catchup failed", "error", err, "did", ai.Did, "i", i, "jobCount", len(job.catchup), "seq", j.evt.Seq) resync = true // fall back to a repo sync break } -- 2.51.2 From 251793e71513d8efc2319cd1be1203eb32a82b33 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 20:41:37 +0000 Subject: [PATCH 067/192] Allow crawling repos that have invalid handles --- bgs/bgs.go | 49 +++++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index eac62775..3e2d7acc 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -26,7 +26,6 @@ import ( lexutil "github.com/bluesky-social/indigo/lex/util" "github.com/bluesky-social/indigo/models" "github.com/bluesky-social/indigo/repomgr" - "github.com/bluesky-social/indigo/util" "github.com/bluesky-social/indigo/xrpc" "golang.org/x/sync/semaphore" "golang.org/x/time/rate" @@ -831,6 +830,21 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event log.Warnw("handle update did not update handle to asserted value", "did", env.RepoHandle.Did, "expected", env.RepoHandle.Handle, "actual", act.Handle) } + // TODO: Update the ReposHandle event type to include "verified" or something + + // Broadcast the handle update to all consumers + err = bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{ + RepoHandle: &comatproto.SyncSubscribeRepos_Handle{ + Did: env.RepoHandle.Did, + Handle: env.RepoHandle.Handle, + Time: env.RepoHandle.Time, + }, + }) + if err != nil { + log.Errorw("failed to broadcast RepoHandle event", "error", err, "did", env.RepoHandle.Did, "handle", env.RepoHandle.Handle) + return fmt.Errorf("failed to broadcast RepoHandle event: %w", err) + } + return nil case env.RepoMigrate != nil: if _, err := bgs.createExternalUser(ctx, env.RepoMigrate.Did); err != nil { @@ -961,13 +975,17 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor handle := hurl.Host + validHandle := true + resdid, err := s.hr.ResolveHandleToDid(ctx, handle) if err != nil { - return nil, fmt.Errorf("failed to resolve users claimed handle (%q) on pds: %w", handle, err) + log.Errorf("failed to resolve users claimed handle (%q) on pds: %s", handle, err) + validHandle = false } if resdid != did { - return nil, fmt.Errorf("claimed handle did not match servers response (%s != %s)", resdid, did) + log.Errorf("claimed handle did not match servers response (%s != %s)", resdid, did) + validHandle = false } s.extUserLk.Lock() @@ -996,17 +1014,6 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor } exu.Handle = handle - - if err := s.events.AddEvent(ctx, &events.XRPCStreamEvent{ - RepoHandle: &comatproto.SyncSubscribeRepos_Handle{ - Did: exu.Did, - Handle: handle, - Time: time.Now().Format(util.ISO8601), - }, - }); err != nil { - // TODO: should we really error here? I'm leaning towards no - return nil, fmt.Errorf("failed to push handle update event: %s", err) - } } return exu, nil } @@ -1017,9 +1024,12 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor // TODO: request this users info from their server to fill out our data... u := User{ - Handle: handle, - Did: did, - PDS: peering.ID, + Did: did, + PDS: peering.ID, + ValidHandle: validHandle, + } + if validHandle { + u.Handle = handle } if err := s.db.Create(&u).Error; err != nil { @@ -1058,11 +1068,14 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor // lets make a local record of that user for the future subj := &models.ActorInfo{ Uid: u.ID, - Handle: handle, DisplayName: "", //*profile.DisplayName, Did: did, Type: "", PDS: peering.ID, + ValidHandle: validHandle, + } + if validHandle { + subj.Handle = handle } if err := s.db.Create(subj).Error; err != nil { return nil, err -- 2.51.2 From 9eca7fbea3cab2200adce41ff31bca5006d9c5d6 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 27 Sep 2023 21:46:14 +0000 Subject: [PATCH 068/192] Use a NullString for Handle --- bgs/bgs.go | 13 +++++++------ events/diskpersist_test.go | 3 ++- indexer/indexer.go | 3 ++- models/models.go | 9 +++++---- pds/handlers.go | 5 +++-- pds/server.go | 3 ++- 6 files changed, 21 insertions(+), 15 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 3e2d7acc..5ac58ab9 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -2,6 +2,7 @@ package bgs import ( "context" + "database/sql" "encoding/json" "errors" "fmt" @@ -436,7 +437,7 @@ type User struct { CreatedAt time.Time UpdatedAt time.Time DeletedAt gorm.DeletedAt `gorm:"index"` - Handle string `gorm:"uniqueIndex"` + Handle sql.NullString `gorm:"uniqueIndex"` Did string `gorm:"uniqueIndex"` PDS uint ValidHandle bool `gorm:"default:true"` @@ -826,7 +827,7 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event return err } - if act.Handle != env.RepoHandle.Handle { + if act.Handle.String != env.RepoHandle.Handle { log.Warnw("handle update did not update handle to asserted value", "did", env.RepoHandle.Did, "expected", env.RepoHandle.Handle, "actual", act.Handle) } @@ -1002,7 +1003,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor } - if exu.Handle != handle { + if exu.Handle.String != handle { // Users handle has changed, update if err := s.db.Model(User{}).Where("id = ?", exu.Uid).Update("handle", handle).Error; err != nil { return nil, fmt.Errorf("failed to update users handle: %w", err) @@ -1013,7 +1014,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor return nil, fmt.Errorf("failed to update actorInfos handle: %w", err) } - exu.Handle = handle + exu.Handle = sql.NullString{String: handle, Valid: true} } return exu, nil } @@ -1029,7 +1030,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor ValidHandle: validHandle, } if validHandle { - u.Handle = handle + u.Handle = sql.NullString{String: handle, Valid: true} } if err := s.db.Create(&u).Error; err != nil { @@ -1075,7 +1076,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor ValidHandle: validHandle, } if validHandle { - subj.Handle = handle + subj.Handle = sql.NullString{String: handle, Valid: true} } if err := s.db.Create(subj).Error; err != nil { return nil, err diff --git a/events/diskpersist_test.go b/events/diskpersist_test.go index 7873dae0..1ac80aab 100644 --- a/events/diskpersist_test.go +++ b/events/diskpersist_test.go @@ -2,6 +2,7 @@ package events_test import ( "context" + "database/sql" "fmt" "os" "path/filepath" @@ -423,7 +424,7 @@ func runTakedownTest(t *testing.T, cs *carstore.CarStore, db *gorm.DB, p events. users[i-1] = &models.ActorInfo{ Uid: i, Did: did, - Handle: handle, + Handle: sql.NullString{String: handle, Valid: true}, } if err := db.Create(&users[i-1]).Error; err != nil { t.Fatal(err) diff --git a/indexer/indexer.go b/indexer/indexer.go index 1eba2275..31d6745c 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -3,6 +3,7 @@ package indexer import ( "bytes" "context" + "database/sql" "errors" "fmt" "sync" @@ -377,7 +378,7 @@ func (ix *Indexer) handleInitActor(ctx context.Context, evt *repomgr.RepoEvent, UpdateAll: true, }).Create(&models.ActorInfo{ Uid: evt.User, - Handle: ai.Handle, + Handle: sql.NullString{String: ai.Handle, Valid: true}, Did: ai.Did, DisplayName: ai.DisplayName, Type: ai.Type, diff --git a/models/models.go b/models/models.go index d856980b..5f4ced29 100644 --- a/models/models.go +++ b/models/models.go @@ -1,6 +1,7 @@ package models import ( + "database/sql" "time" "gorm.io/gorm" @@ -35,8 +36,8 @@ type RepostRecord struct { type ActorInfo struct { gorm.Model - Uid Uid `gorm:"uniqueindex"` - Handle string `gorm:"uniqueindex"` + Uid Uid `gorm:"uniqueindex"` + Handle sql.NullString `gorm:"uniqueindex"` DisplayName string Did string `gorm:"uniqueindex"` Following int64 @@ -50,7 +51,7 @@ type ActorInfo struct { func (ai *ActorInfo) ActorRef() *bsky.ActorDefs_ProfileViewBasic { return &bsky.ActorDefs_ProfileViewBasic{ Did: ai.Did, - Handle: ai.Handle, + Handle: ai.Handle.String, DisplayName: &ai.DisplayName, } } @@ -59,7 +60,7 @@ func (ai *ActorInfo) ActorRef() *bsky.ActorDefs_ProfileViewBasic { func (ai *ActorInfo) ActorView() *bsky.ActorDefs_ProfileView { return &bsky.ActorDefs_ProfileView{ Did: ai.Did, - Handle: ai.Handle, + Handle: ai.Handle.String, DisplayName: &ai.DisplayName, } } diff --git a/pds/handlers.go b/pds/handlers.go index b99eba16..46a58454 100644 --- a/pds/handlers.go +++ b/pds/handlers.go @@ -3,6 +3,7 @@ package pds import ( "bytes" "context" + "database/sql" "fmt" "io" "time" @@ -27,7 +28,7 @@ func (s *Server) handleAppBskyActorGetProfile(ctx context.Context, actor string) Description: nil, PostsCount: &profile.Posts, FollowsCount: &profile.Following, - Handle: profile.Handle, + Handle: profile.Handle.String, DisplayName: &profile.DisplayName, FollowersCount: &profile.Followers, }, nil @@ -326,7 +327,7 @@ func (s *Server) handleComAtprotoServerCreateAccount(ctx context.Context, body * ai := &models.ActorInfo{ Uid: u.ID, Did: d, - Handle: body.Handle, + Handle: sql.NullString{String: body.Handle, Valid: true}, } if err := s.db.Create(ai).Error; err != nil { return nil, err diff --git a/pds/server.go b/pds/server.go index b9f98a09..272e0e32 100644 --- a/pds/server.go +++ b/pds/server.go @@ -2,6 +2,7 @@ package pds import ( "context" + "database/sql" "encoding/json" "errors" "fmt" @@ -224,7 +225,7 @@ func (s *Server) createExternalUser(ctx context.Context, did string) (*models.Ac // lets make a local record of that user for the future subj := &models.ActorInfo{ Uid: u.ID, - Handle: handle, + Handle: sql.NullString{String: handle, Valid: true}, DisplayName: *profile.DisplayName, Did: did, Type: "", -- 2.51.2 From 6fe2803a4cff472b68ebcf5e8545db3d1d9d506d Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 16:26:21 +0000 Subject: [PATCH 069/192] Fix nil pointer deref --- repomgr/repomgr.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index 347cf66d..5b11645e 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -885,12 +885,19 @@ func (rm *RepoManager) processNewRepo(ctx context.Context, user models.Uid, r io } if err := cb(ctx, root, finish, ds); err != nil { - return fmt.Errorf("cb errored root: %s, rev: %s: %w", root, *rev, err) + return fmt.Errorf("cb errored root: %s, rev: %s: %w", root, stringOrNil(rev), err) } return nil } +func stringOrNil(s *string) string { + if s == nil { + return "nil" + } + return *s +} + // walkTree returns all cids linked recursively by the root, skipping any cids // in the 'skip' map, and not erroring on 'not found' if prevMissing is set func walkTree(ctx context.Context, skip map[cid.Cid]bool, root cid.Cid, bs blockstore.Blockstore, prevMissing bool) ([]cid.Cid, error) { -- 2.51.2 From eecbf9088488a4b680161c5a31d67af313fc6fc0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 28 Sep 2023 11:30:38 -0700 Subject: [PATCH 070/192] for now, add duplicate stale refs to the keep set --- bgs/bgs.go | 7 +++++++ carstore/bs.go | 21 ++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 5ac58ab9..c3f22fad 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -281,6 +281,13 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { sendHeader = false } + if strings.HasPrefix(ctx.Path(), "/admin/") { + ctx.JSON(500, map[string]any{ + "error": err.Error(), + }) + return + } + log.Warnf("HANDLER ERROR: (%s) %s", ctx.Path(), err) if sendHeader { diff --git a/carstore/bs.go b/carstore/bs.go index 28823618..027542a0 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -23,6 +23,7 @@ import ( cbor "github.com/ipfs/go-ipld-cbor" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-libipfs/blocks" + logging "github.com/ipfs/go-log" car "github.com/ipld/go-car" carutil "github.com/ipld/go-car/util" cbg "github.com/whyrusleeping/cbor-gen" @@ -31,6 +32,8 @@ import ( "gorm.io/gorm" ) +var log = logging.Logger("carstore") + const MaxSliceLength = 2 << 20 type CarStore struct { @@ -1180,6 +1183,7 @@ type CompactionStats struct { SkippedShards int `json:"skippedShards"` ShardsDeleted int `json:"shardsDeleted"` RefsDeleted int `json:"refsDeleted"` + DupeCount int `json:"dupeCount"` } func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*CompactionStats, error) { @@ -1214,13 +1218,16 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } stale := make(map[cid.Cid]bool) + var dupes []cid.Cid var hasDirtyDupes bool for _, br := range staleRefs { if stale[br.Cid.CID] { + delete(stale, br.Cid.CID) // remove dupes from stale list, see comment below hasDirtyDupes = true - break + dupes = append(dupes, br.Cid.CID) + } else { + stale[br.Cid.CID] = true } - stale[br.Cid.CID] = true } if hasDirtyDupes { @@ -1233,7 +1240,10 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co // focus on compacting everything else. it leaves *some* dirty blocks // still around but we're doing that anyways since compaction isnt a // perfect process - return nil, fmt.Errorf("WIP: not currently handling this case") + + log.Warnw("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs)) + + //return nil, fmt.Errorf("WIP: not currently handling this case") } keep := make(map[cid.Cid]bool) @@ -1243,6 +1253,10 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } } + for _, dupe := range dupes { + keep[dupe] = true + } + results := aggrRefs(brefs, shardsById, stale) var sum int for _, r := range results { @@ -1335,6 +1349,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } stats.RefsDeleted = num + stats.DupeCount = len(dupes) return stats, nil } -- 2.51.2 From 97850b81a4ddf7b14767d7520d36524abf73d4f7 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 28 Sep 2023 14:01:00 -0700 Subject: [PATCH 071/192] use background contexts for repo compaction admin routes --- bgs/admin.go | 7 +++++-- carstore/bs.go | 7 +++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index 3e8d92de..547d83fa 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -12,6 +12,7 @@ import ( "github.com/bluesky-social/indigo/models" "github.com/labstack/echo/v4" dto "github.com/prometheus/client_model/go" + "go.opentelemetry.io/otel" "golang.org/x/time/rate" "gorm.io/gorm" ) @@ -397,7 +398,8 @@ func (bgs *BGS) handleAdminChangePDSCrawlLimit(e echo.Context) error { } func (bgs *BGS) handleAdminCompactRepo(e echo.Context) error { - ctx := e.Request().Context() + ctx, span := otel.Tracer("bgs").Start(context.Background(), "adminCompactRepo") + defer span.End() did := e.QueryParam("did") if did == "" { @@ -421,7 +423,8 @@ func (bgs *BGS) handleAdminCompactRepo(e echo.Context) error { } func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { - ctx := e.Request().Context() + ctx, span := otel.Tracer("bgs").Start(context.Background(), "adminCompactAllRepos") + defer span.End() if err := bgs.runRepoCompaction(ctx); err != nil { return fmt.Errorf("compaction run failed: %w", err) diff --git a/carstore/bs.go b/carstore/bs.go index 027542a0..74833756 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1447,6 +1447,13 @@ func (cs *CarStore) compactBucket(ctx context.Context, user models.Uid, b *compB } if err := cs.putShard(ctx, &shard, nbrefs, nil, true); err != nil { + // if writing the shard fails, we should also delete the file + _ = fi.Close() + + if err2 := os.Remove(fi.Name()); err2 != nil { + log.Errorf("failed to remove shard file (%s) after failed db transaction: %w", fi.Name(), err2) + } + return err } return nil -- 2.51.2 From 61c8c55c82842f84f0f9111ed4b721bde9a0463e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 29 Sep 2023 10:06:06 -0700 Subject: [PATCH 072/192] add endpoint to reset a given repo, other tweaks to compaction --- bgs/admin.go | 24 ++++++++++++++++++++++++ bgs/bgs.go | 1 + carstore/bs.go | 13 +++++++++++-- repomgr/repomgr.go | 10 +++++++++- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index 547d83fa..71697bbd 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -484,3 +484,27 @@ func (bgs *BGS) handleAdminGetResyncPDS(e echo.Context) error { "resync": resync, }) } + +func (bgs *BGS) handleAdminResetRepo(e echo.Context) error { + ctx := e.Request().Context() + + did := e.QueryParam("did") + if did == "" { + return fmt.Errorf("must pass a did") + } + + ai, err := bgs.Index.LookupUserByDid(ctx, did) + if err != nil { + return fmt.Errorf("no such user: %w", err) + } + + if err := bgs.repoman.ResetRepo(ctx, ai.Uid); err != nil { + return err + } + + if err := bgs.Index.Crawler.Crawl(ctx, ai); err != nil { + return err + } + + return nil +} diff --git a/bgs/bgs.go b/bgs/bgs.go index c3f22fad..8de29840 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -333,6 +333,7 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { admin.POST("/repo/reverseTakedown", bgs.handleAdminReverseTakedown) admin.POST("/repo/compact", bgs.handleAdminCompactRepo) admin.POST("/repo/compactAll", bgs.handleAdminCompactAllRepos) + admin.POST("/repo/reset", bgs.handleAdminResetRepo) // PDS-related Admin API admin.GET("/pds/list", bgs.handleListPDSs) diff --git a/carstore/bs.go b/carstore/bs.go index 74833756..a5d46da7 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -951,7 +951,7 @@ func (cs *CarStore) Stat(ctx context.Context, usr models.Uid) ([]UserStat, error return out, nil } -func (cs *CarStore) TakeDownRepo(ctx context.Context, user models.Uid) error { +func (cs *CarStore) WipeUserData(ctx context.Context, user models.Uid) error { var shards []*CarShard if err := cs.meta.Find(&shards, "usr = ?", user).Error; err != nil { return err @@ -1222,7 +1222,6 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co var hasDirtyDupes bool for _, br := range staleRefs { if stale[br.Cid.CID] { - delete(stale, br.Cid.CID) // remove dupes from stale list, see comment below hasDirtyDupes = true dupes = append(dupes, br.Cid.CID) } else { @@ -1230,6 +1229,10 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } } + for _, dupe := range dupes { + delete(stale, dupe) // remove dupes from stale list, see comment below + } + if hasDirtyDupes { // if we have no duplicates, then the keep set is simply all the 'clean' blockRefs // in the case we have duplicate dirty references we have to compute @@ -1415,9 +1418,14 @@ func (cs *CarStore) compactBucket(ctx context.Context, user models.Uid, b *compB offset := hnw var nbrefs []map[string]any + written := make(map[cid.Cid]bool) for _, s := range b.shards { sh := shardsById[s.ID] if err := cs.iterateShardBlocks(ctx, &sh, func(blk blockformat.Block) error { + if written[blk.Cid()] { + return nil + } + if keep[blk.Cid()] { nw, err := LdWrite(fi, blk.Cid().Bytes(), blk.RawData()) if err != nil { @@ -1430,6 +1438,7 @@ func (cs *CarStore) compactBucket(ctx context.Context, user models.Uid, b *compB }) offset += nw + written[blk.Cid()] = true } return nil }); err != nil { diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index 5b11645e..e98b5a4c 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -951,5 +951,13 @@ func (rm *RepoManager) TakeDownRepo(ctx context.Context, uid models.Uid) error { unlock := rm.lockUser(ctx, uid) defer unlock() - return rm.cs.TakeDownRepo(ctx, uid) + return rm.cs.WipeUserData(ctx, uid) +} + +// technically identical to TakeDownRepo, for now +func (rm *RepoManager) ResetRepo(ctx context.Context, uid models.Uid) error { + unlock := rm.lockUser(ctx, uid) + defer unlock() + + return rm.cs.WipeUserData(ctx, uid) } -- 2.51.2 From 9cd705933aa1ae29f4e646c003bff2c6e483b6ba Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 29 Sep 2023 10:31:41 -0700 Subject: [PATCH 073/192] cli command to trigger reset repo --- cmd/gosky/bgs.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/cmd/gosky/bgs.go b/cmd/gosky/bgs.go index 727f56f3..dfbddb68 100644 --- a/cmd/gosky/bgs.go +++ b/cmd/gosky/bgs.go @@ -33,6 +33,7 @@ var bgsAdminCmd = &cli.Command{ bgsSetNewSubsEnabledCmd, bgsCompactRepo, bgsCompactAll, + bgsResetRepo, }, } @@ -381,3 +382,44 @@ var bgsCompactAll = &cli.Command{ return nil }, } + +var bgsResetRepo = &cli.Command{ + Name: "reset-repo", + Action: func(cctx *cli.Context) error { + url := cctx.String("bgs") + "/admin/repo/reset" + + did := cctx.Args().First() + url += fmt.Sprintf("?did=%s", did) + + req, err := http.NewRequest("POST", url, nil) + if err != nil { + return err + } + + auth := cctx.String("key") + req.Header.Set("Authorization", "Bearer "+auth) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + + if resp.StatusCode != 200 { + var e xrpc.XRPCError + if err := json.NewDecoder(resp.Body).Decode(&e); err != nil { + return err + } + + return &e + } + + var out map[string]any + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return err + } + + fmt.Println(out) + + return nil + }, +} -- 2.51.2 From b320b8a17b67cc59574ce490f2656c2cedf8ed1e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 29 Sep 2023 11:36:27 -0700 Subject: [PATCH 074/192] drop shard cache when wiping user data from carstore --- bgs/admin.go | 4 +++- bgs/bgs.go | 4 ++-- carstore/bs.go | 9 +++++++++ repomgr/repomgr.go | 2 +- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index 71697bbd..13504125 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -506,5 +506,7 @@ func (bgs *BGS) handleAdminResetRepo(e echo.Context) error { return err } - return nil + return e.JSON(200, map[string]any{ + "success": true, + }) } diff --git a/bgs/bgs.go b/bgs/bgs.go index 8de29840..1c359fa2 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -281,6 +281,8 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { sendHeader = false } + log.Warnf("HANDLER ERROR: (%s) %s", ctx.Path(), err) + if strings.HasPrefix(ctx.Path(), "/admin/") { ctx.JSON(500, map[string]any{ "error": err.Error(), @@ -288,8 +290,6 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { return } - log.Warnf("HANDLER ERROR: (%s) %s", ctx.Path(), err) - if sendHeader { ctx.Response().WriteHeader(500) } diff --git a/carstore/bs.go b/carstore/bs.go index a5d46da7..cf2e2a8d 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -276,6 +276,13 @@ func (cs *CarStore) checkLastShardCache(user models.Uid) *CarShard { return nil } +func (cs *CarStore) removeLastShardCache(user models.Uid) { + cs.lscLk.Lock() + defer cs.lscLk.Unlock() + + delete(cs.lastShardCache, user) +} + func (cs *CarStore) putLastShardCache(ls *CarShard) { cs.lscLk.Lock() defer cs.lscLk.Unlock() @@ -963,6 +970,8 @@ func (cs *CarStore) WipeUserData(ctx context.Context, user models.Uid) error { } } + cs.removeLastShardCache(user) + return nil } diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index e98b5a4c..e9ffece4 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -731,7 +731,7 @@ func (rm *RepoManager) ImportNewRepo(ctx context.Context, user models.Uid, repoD diffops, err := r.DiffSince(ctx, curhead) if err != nil { - return fmt.Errorf("diff trees: %w", err) + return fmt.Errorf("diff trees (curhead: %s): %w", curhead, err) } var ops []RepoOp -- 2.51.2 From 0c7204db93582728e58057fb5f5545435642ead8 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 19:06:39 +0000 Subject: [PATCH 075/192] Clean up coalescing code --- atproto/identity/cache_directory.go | 109 ++++++++++------------------ 1 file changed, 39 insertions(+), 70 deletions(-) diff --git a/atproto/identity/cache_directory.go b/atproto/identity/cache_directory.go index d570cd4d..900a86b8 100644 --- a/atproto/identity/cache_directory.go +++ b/atproto/identity/cache_directory.go @@ -118,18 +118,24 @@ func (d *CacheDirectory) updateHandle(ctx context.Context, h syntax.Handle) (*Ha return &he, nil } -func (d *CacheDirectory) coalescedResolveHandle(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { - resC := make(chan struct{}, 1) - actualLookup := false +func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { + entry, ok := d.handleCache.Get(h) + if ok && !d.IsHandleStale(&entry) { + handleCacheHits.Inc() + return entry.DID, entry.Err + } + handleCacheMisses.Inc() - val, loaded := d.handleLookupChans.LoadOrStore(handle.String(), resC) + // Coalesce multiple requests for the same Handle + res := make(chan struct{}) + val, loaded := d.handleLookupChans.LoadOrStore(h.String(), res) if loaded { handleRequestsCoalesced.Inc() // Wait for the result from the original goroutine select { case <-val.(chan struct{}): // The result should now be in the cache - entry, ok := d.handleCache.Get(handle) + entry, ok := d.handleCache.Get(h) if ok && !d.IsHandleStale(&entry) { return entry.DID, entry.Err } @@ -137,45 +143,25 @@ func (d *CacheDirectory) coalescedResolveHandle(ctx context.Context, handle synt case <-ctx.Done(): return "", ctx.Err() } - } else { - actualLookup = true } var did syntax.DID - var err error // Perform actual lookup only if this goroutine is the one doing it - if actualLookup { - var entry *HandleEntry - // Update the cache entry for this Handle - entry, err = d.updateHandle(ctx, handle) - if err == nil && entry != nil { - did = entry.DID - } - // Cleanup the coalesce map and close the results channel - d.handleLookupChans.Delete(handle.String()) - // Callers waiting will now get the result from the cache - close(resC) + var newEntry *HandleEntry + // Update the cache entry for this Handle + newEntry, err := d.updateHandle(ctx, h) + if err == nil && newEntry != nil { + did = newEntry.DID } + // Cleanup the coalesce map and close the results channel + d.handleLookupChans.Delete(h.String()) + // Callers waiting will now get the result from the cache + close(res) return did, err } -func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (syntax.DID, error) { - entry, ok := d.handleCache.Get(h) - if ok && !d.IsHandleStale(&entry) { - handleCacheHits.Inc() - return entry.DID, entry.Err - } - handleCacheMisses.Inc() - - did, err := d.coalescedResolveHandle(ctx, h) - if err != nil { - return "", err - } - return did, nil -} - func (d *CacheDirectory) updateDID(ctx context.Context, did syntax.DID) (*IdentityEntry, error) { ident, err := d.Inner.LookupDID(ctx, did) // persist the identity lookup error, instead of processing it immediately @@ -201,14 +187,20 @@ func (d *CacheDirectory) updateDID(ctx context.Context, did syntax.DID) (*Identi return &entry, nil } -func (d *CacheDirectory) coalescedResolveDID(ctx context.Context, did syntax.DID) (*Identity, error) { - resC := make(chan struct{}, 1) - actualLookup := false +func (d *CacheDirectory) LookupDID(ctx context.Context, did syntax.DID) (*Identity, error) { + entry, ok := d.identityCache.Get(did) + if ok && !d.IsIdentityStale(&entry) { + identityCacheHits.Inc() + return entry.Identity, entry.Err + } + identityCacheMisses.Inc() - val, loaded := d.didLookupChans.LoadOrStore(did.String(), resC) + // Coalesce multiple requests for the same DID + res := make(chan struct{}) + val, loaded := d.didLookupChans.LoadOrStore(did.String(), res) if loaded { identityRequestsCoalesced.Inc() - // Wait for the result from the original goroutine + // Wait for the result from the pending request select { case <-val.(chan struct{}): // The result should now be in the cache @@ -220,45 +212,22 @@ func (d *CacheDirectory) coalescedResolveDID(ctx context.Context, did syntax.DID case <-ctx.Done(): return nil, ctx.Err() } - } else { - actualLookup = true } var doc *Identity - var err error - - // Perform actual lookup only if this goroutine is the one doing it - if actualLookup { - var entry *IdentityEntry - // Update the cache entry for this DID - entry, err = d.updateDID(ctx, did) - if err == nil && entry != nil { - doc = entry.Identity - } - // Cleanup the coalesce map and close the results channel - d.didLookupChans.Delete(did.String()) - // Callers waiting will now get the result from the cache - close(resC) + // Update the Identity Entry from PLC and cache the result + newEntry, err := d.updateDID(ctx, did) + if err == nil && newEntry != nil { + doc = newEntry.Identity } + // Cleanup the coalesce map and close the results channel + d.didLookupChans.Delete(did.String()) + // Callers waiting will now get the result from the cache + close(res) return doc, err } -func (d *CacheDirectory) LookupDID(ctx context.Context, did syntax.DID) (*Identity, error) { - entry, ok := d.identityCache.Get(did) - if ok && !d.IsIdentityStale(&entry) { - identityCacheHits.Inc() - return entry.Identity, entry.Err - } - identityCacheMisses.Inc() - - doc, err := d.coalescedResolveDID(ctx, did) - if err != nil { - return nil, err - } - return doc, nil -} - func (d *CacheDirectory) LookupHandle(ctx context.Context, h syntax.Handle) (*Identity, error) { did, err := d.ResolveHandle(ctx, h) if err != nil { -- 2.51.2 From 355098bcdd49a7b47b4991bd0d7006a7285a80e1 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 19:07:44 +0000 Subject: [PATCH 076/192] Fix comment --- atproto/identity/cache_directory.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atproto/identity/cache_directory.go b/atproto/identity/cache_directory.go index 900a86b8..c267b8e1 100644 --- a/atproto/identity/cache_directory.go +++ b/atproto/identity/cache_directory.go @@ -131,7 +131,7 @@ func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (sy val, loaded := d.handleLookupChans.LoadOrStore(h.String(), res) if loaded { handleRequestsCoalesced.Inc() - // Wait for the result from the original goroutine + // Wait for the result from the pending request select { case <-val.(chan struct{}): // The result should now be in the cache -- 2.51.2 From 35a9850e3b0281daa8ab20658bde90499a7d01f8 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 19:19:53 +0000 Subject: [PATCH 077/192] cleaning --- atproto/identity/cache_directory.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/atproto/identity/cache_directory.go b/atproto/identity/cache_directory.go index c267b8e1..15e83cb1 100644 --- a/atproto/identity/cache_directory.go +++ b/atproto/identity/cache_directory.go @@ -146,10 +146,7 @@ func (d *CacheDirectory) ResolveHandle(ctx context.Context, h syntax.Handle) (sy } var did syntax.DID - - // Perform actual lookup only if this goroutine is the one doing it - var newEntry *HandleEntry - // Update the cache entry for this Handle + // Update the Handle Entry from PLC and cache the result newEntry, err := d.updateHandle(ctx, h) if err == nil && newEntry != nil { did = newEntry.DID -- 2.51.2 From 22d5775f091e96e82fdd270e7a989faee992cbdb Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 21:40:22 +0000 Subject: [PATCH 078/192] Update default netsync checkout to BGS --- cmd/netsync/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/netsync/main.go b/cmd/netsync/main.go index cc0a4e08..d316f5b1 100644 --- a/cmd/netsync/main.go +++ b/cmd/netsync/main.go @@ -69,7 +69,7 @@ func main() { &cli.StringFlag{ Name: "checkout-path", Usage: "path to checkout endpoint", - Value: "https://bsky.social/xrpc/com.atproto.sync.getCheckout", + Value: "https://bgs.bsky.social/xrpc/com.atproto.sync.getRepo", }, &cli.StringFlag{ Name: "magic-header-key", -- 2.51.2 From afb75f3b48fbe7556aba002d80d5126518b0f680 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 22:13:27 +0000 Subject: [PATCH 079/192] Provide helpful error messages and status codes for BGS endpoints --- bgs/handlers.go | 50 +++++++++++++++++++++++++------------------------ blobs/blobs.go | 11 +++++++++++ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/bgs/handlers.go b/bgs/handlers.go index dc282717..143ee148 100644 --- a/bgs/handlers.go +++ b/bgs/handlers.go @@ -12,6 +12,8 @@ import ( atproto "github.com/bluesky-social/indigo/api/atproto" comatprototypes "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/blobs" + "github.com/bluesky-social/indigo/mst" "gorm.io/gorm" "github.com/bluesky-social/indigo/util" @@ -33,19 +35,22 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri if commit != "" { reqCid, err = cid.Decode(commit) if err != nil { - return nil, fmt.Errorf("failed to decode commit cid: %w", err) + return nil, echo.NewHTTPError(http.StatusBadRequest, "failed to decode commit cid") } } _, record, err := s.repoman.GetRecord(ctx, u.Uid, collection, rkey, reqCid) if err != nil { - return nil, fmt.Errorf("failed to get record: %w", err) + if errors.Is(err, mst.ErrNotFound) { + return nil, echo.NewHTTPError(http.StatusNotFound, "record not found in repo") + } + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get record from repo") } buf := new(bytes.Buffer) err = record.MarshalCBOR(buf) if err != nil { - return nil, fmt.Errorf("failed to marshal record: %w", err) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to marshal record to CBOR") } return buf, nil @@ -63,7 +68,7 @@ func (s *BGS) handleComAtprotoSyncGetRepo(ctx context.Context, did string, since // TODO: stream the response buf := new(bytes.Buffer) if err := s.repoman.ReadRepo(ctx, u.Uid, since, buf); err != nil { - return nil, fmt.Errorf("failed to read repo: %w", err) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to read repo into buffer") } return buf, nil @@ -76,27 +81,21 @@ func (s *BGS) handleComAtprotoSyncGetBlocks(ctx context.Context, cids []string, func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatprototypes.SyncRequestCrawl_Input) error { host := body.Hostname if host == "" { - return fmt.Errorf("must pass valid hostname") + return echo.NewHTTPError(http.StatusBadRequest, "must pass hostname") } if strings.HasPrefix(host, "https://") || strings.HasPrefix(host, "http://") { - return &echo.HTTPError{ - Code: 400, - Message: "must pass domain without protocol scheme", - } + return echo.NewHTTPError(http.StatusBadRequest, "must pass domain without protocol scheme") } norm, err := util.NormalizeHostname(host) if err != nil { - return err + return echo.NewHTTPError(http.StatusBadRequest, "failed to normalize hostname") } banned, err := s.domainIsBanned(ctx, host) if banned { - return &echo.HTTPError{ - Code: 401, - Message: "domain is banned", - } + return echo.NewHTTPError(http.StatusForbidden, "domain is banned") } log.Warnf("TODO: better host validation for crawl requests") @@ -112,10 +111,7 @@ func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatp desc, err := atproto.ServerDescribeServer(ctx, c) if err != nil { - return &echo.HTTPError{ - Code: 401, - Message: fmt.Sprintf("given host failed to respond to ping: %s", err), - } + return echo.NewHTTPError(http.StatusInternalServerError, "requested host failed to respond to describe request") } // Maybe we could do something with this response later @@ -136,7 +132,10 @@ func (s *BGS) handleComAtprotoSyncGetBlob(ctx context.Context, cid string, did s b, err := s.blobs.GetBlob(ctx, cid, did) if err != nil { - return nil, err + if errors.Is(err, blobs.NotFoundErr) { + return nil, echo.NewHTTPError(http.StatusNotFound, "blob not found") + } + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get blob") } return bytes.NewReader(b), nil @@ -153,7 +152,7 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, if cursor != "" { c, err = strconv.ParseInt(cursor, 10, 64) if err != nil { - return nil, fmt.Errorf("invalid cursor: %w", err) + return nil, echo.NewHTTPError(http.StatusBadRequest, "couldn't parse your cursor as an integer") } } @@ -162,7 +161,7 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, if err == gorm.ErrRecordNotFound { return &comatprototypes.SyncListRepos_Output{}, nil } - return nil, fmt.Errorf("failed to get users: %w", err) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to query users") } if len(users) == 0 { @@ -177,7 +176,8 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, user := users[i] root, err := s.repoman.GetRepoRoot(ctx, user.ID) if err != nil { - return nil, fmt.Errorf("failed to get repo root for (%s): %w", user.Did, err) + log.Errorw("failed to get repo root", "err", err, "did", user.Did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, fmt.Sprintf("failed to get repo root for (%s): %v", user.Did, err.Error())) } resp.Repos = append(resp.Repos, &comatprototypes.SyncListRepos_Repo{ @@ -204,12 +204,14 @@ func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context, did strin root, err := s.repoman.GetRepoRoot(ctx, u.Uid) if err != nil { - return nil, fmt.Errorf("failed to get repo root: %w", err) + log.Errorw("failed to get repo root", "err", err, "did", u.Did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get repo root") } rev, err := s.repoman.GetRepoRev(ctx, u.Uid) if err != nil { - return nil, fmt.Errorf("failed to get repo rev: %w", err) + log.Errorw("failed to get repo rev", "err", err, "did", u.Did) + return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get repo rev") } return &comatprototypes.SyncGetLatestCommit_Output{ diff --git a/blobs/blobs.go b/blobs/blobs.go index 23848067..ca199c9d 100644 --- a/blobs/blobs.go +++ b/blobs/blobs.go @@ -2,10 +2,13 @@ package blobs import ( "context" + "fmt" "os" "path/filepath" ) +var NotFoundErr = fmt.Errorf("blob not found") + type BlobStore interface { PutBlob(ctx context.Context, cid string, did string, blob []byte) error GetBlob(ctx context.Context, cid string, did string) ([]byte, error) @@ -25,5 +28,13 @@ func (dbs *DiskBlobStore) PutBlob(ctx context.Context, cid string, did string, b } func (dbs *DiskBlobStore) GetBlob(ctx context.Context, cid string, did string) ([]byte, error) { + // Check if the blob exists + _, err := os.Stat(filepath.Join(dbs.Dir, did, cid)) + if err != nil { + if os.IsNotExist(err) { + return nil, NotFoundErr + } + return nil, err + } return os.ReadFile(filepath.Join(dbs.Dir, did, cid)) } -- 2.51.2 From 9955111ad02681f05bd2455445ba619b7972f6ee Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 22:15:58 +0000 Subject: [PATCH 080/192] Log errors that aren't the user's fault --- bgs/handlers.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bgs/handlers.go b/bgs/handlers.go index 143ee148..c2920188 100644 --- a/bgs/handlers.go +++ b/bgs/handlers.go @@ -28,6 +28,7 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri if errors.Is(err, gorm.ErrRecordNotFound) { return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") } + log.Errorw("failed to lookup user", "err", err, "did", did) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") } @@ -35,6 +36,7 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri if commit != "" { reqCid, err = cid.Decode(commit) if err != nil { + log.Errorw("failed to decode commit cid", "err", err, "cid", commit) return nil, echo.NewHTTPError(http.StatusBadRequest, "failed to decode commit cid") } } @@ -44,12 +46,14 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri if errors.Is(err, mst.ErrNotFound) { return nil, echo.NewHTTPError(http.StatusNotFound, "record not found in repo") } + log.Errorw("failed to get record from repo", "err", err, "did", did, "collection", collection, "rkey", rkey) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get record from repo") } buf := new(bytes.Buffer) err = record.MarshalCBOR(buf) if err != nil { + log.Errorw("failed to marshal record to CBOR", "err", err, "did", did, "collection", collection, "rkey", rkey) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to marshal record to CBOR") } @@ -62,12 +66,14 @@ func (s *BGS) handleComAtprotoSyncGetRepo(ctx context.Context, did string, since if errors.Is(err, gorm.ErrRecordNotFound) { return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") } + log.Errorw("failed to lookup user", "err", err, "did", did) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") } // TODO: stream the response buf := new(bytes.Buffer) if err := s.repoman.ReadRepo(ctx, u.Uid, since, buf); err != nil { + log.Errorw("failed to read repo into buffer", "err", err, "did", did) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to read repo into buffer") } @@ -135,6 +141,7 @@ func (s *BGS) handleComAtprotoSyncGetBlob(ctx context.Context, cid string, did s if errors.Is(err, blobs.NotFoundErr) { return nil, echo.NewHTTPError(http.StatusNotFound, "blob not found") } + log.Errorw("failed to get blob", "err", err, "cid", cid, "did", did) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to get blob") } @@ -161,6 +168,7 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, if err == gorm.ErrRecordNotFound { return &comatprototypes.SyncListRepos_Output{}, nil } + log.Errorw("failed to query users", "err", err) return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to query users") } -- 2.51.2 From 176fdf8b565a0128de804b1a28ab7b2f06c69e27 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 29 Sep 2023 22:29:15 +0000 Subject: [PATCH 081/192] Fix domain ban test --- bgs/handlers.go | 4 ++-- testing/integ_test.go | 14 +++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/bgs/handlers.go b/bgs/handlers.go index c2920188..a278e93f 100644 --- a/bgs/handlers.go +++ b/bgs/handlers.go @@ -101,7 +101,7 @@ func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatp banned, err := s.domainIsBanned(ctx, host) if banned { - return echo.NewHTTPError(http.StatusForbidden, "domain is banned") + return echo.NewHTTPError(http.StatusUnauthorized, "domain is banned") } log.Warnf("TODO: better host validation for crawl requests") @@ -117,7 +117,7 @@ func (s *BGS) handleComAtprotoSyncRequestCrawl(ctx context.Context, body *comatp desc, err := atproto.ServerDescribeServer(ctx, c) if err != nil { - return echo.NewHTTPError(http.StatusInternalServerError, "requested host failed to respond to describe request") + return echo.NewHTTPError(http.StatusBadRequest, "requested host failed to respond to describe request") } // Maybe we could do something with this response later diff --git a/testing/integ_test.go b/testing/integ_test.go index b285f876..67b94633 100644 --- a/testing/integ_test.go +++ b/testing/integ_test.go @@ -380,18 +380,22 @@ func TestDomainBans(t *testing.T) { t.Fatal("domain should be banned") } - if err := atproto.SyncRequestCrawl(context.TODO(), c, &atproto.SyncRequestCrawl_Input{Hostname: "app.pds.foo.com"}); err == nil { + err := atproto.SyncRequestCrawl(context.TODO(), c, &atproto.SyncRequestCrawl_Input{Hostname: "app.pds.foo.com"}) + if err == nil { t.Fatal("domain should be banned") } + if !strings.Contains(err.Error(), "XRPC ERROR 401") { + t.Fatal("should have failed with a 401") + } + // should not be banned - err := atproto.SyncRequestCrawl(context.TODO(), c, &atproto.SyncRequestCrawl_Input{Hostname: "foo.bar.com"}) + err = atproto.SyncRequestCrawl(context.TODO(), c, &atproto.SyncRequestCrawl_Input{Hostname: "foo.bar.com"}) if err == nil { t.Fatal("should still fail") } - if !strings.Contains(err.Error(), "XRPC ERROR 401") { - t.Fatal("should have failed with a 401") + if !strings.Contains(err.Error(), "XRPC ERROR 400") { + t.Fatal("should have failed with a 400") } - } -- 2.51.2 From c492597e9c3897d1ee67eed176000e2c72bb429d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 29 Sep 2023 15:44:17 -0700 Subject: [PATCH 082/192] properly account for dupe-ref staleRefs --- carstore/bs.go | 22 ++++++++++------ carstore/repo_test.go | 59 +++++++++++++++++++++++++++++++------------ cmd/gosky/debug.go | 2 +- mst/mst.go | 8 +++--- 4 files changed, 62 insertions(+), 29 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index cf2e2a8d..e646df8f 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -751,12 +751,11 @@ func generateInsertQuery(data []map[string]any) (string, []any) { // Function to create in batches func createInBatches(ctx context.Context, tx *gorm.DB, data []map[string]any, batchSize int) error { for i := 0; i < len(data); i += batchSize { - end := i + batchSize - if end > len(data) { - end = len(data) + batch := data[i:] + if len(batch) > batchSize { + batch = batch[:batchSize] } - batch := data[i:end] query, values := generateInsertQuery(batch) if err := tx.WithContext(ctx).Exec(query, values...).Error; err != nil { @@ -1227,14 +1226,21 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } stale := make(map[cid.Cid]bool) + for _, br := range staleRefs { + stale[br.Cid.CID] = true + } + + // if we have a staleRef that references multiple blockRefs, we consider that block a 'dirty duplicate' var dupes []cid.Cid var hasDirtyDupes bool - for _, br := range staleRefs { - if stale[br.Cid.CID] { - hasDirtyDupes = true + seenBlocks := make(map[cid.Cid]bool) + for _, br := range brefs { + if seenBlocks[br.Cid.CID] { dupes = append(dupes, br.Cid.CID) + hasDirtyDupes = true + delete(stale, br.Cid.CID) } else { - stale[br.Cid.CID] = true + seenBlocks[br.Cid.CID] = true } } diff --git a/carstore/repo_test.go b/carstore/repo_test.go index 5ce81ece..1093f459 100644 --- a/carstore/repo_test.go +++ b/carstore/repo_test.go @@ -3,6 +3,7 @@ package carstore import ( "bytes" "context" + "errors" "fmt" "io" "os" @@ -17,6 +18,7 @@ import ( "github.com/ipfs/go-cid" flatfs "github.com/ipfs/go-ds-flatfs" blockstore "github.com/ipfs/go-ipfs-blockstore" + ipld "github.com/ipfs/go-ipld-format" "gorm.io/driver/sqlite" "gorm.io/gorm" ) @@ -138,7 +140,7 @@ func TestBasicOperation(t *testing.T) { if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { t.Fatal(err) } - checkRepo(t, buf, recs) + checkRepo(t, cs, buf, recs) if _, err := cs.CompactUserShards(ctx, 1); err != nil { t.Fatal(err) @@ -148,7 +150,7 @@ func TestBasicOperation(t *testing.T) { if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { t.Fatal(err) } - checkRepo(t, buf, recs) + checkRepo(t, cs, buf, recs) } func TestRepeatedCompactions(t *testing.T) { @@ -177,8 +179,10 @@ func TestRepeatedCompactions(t *testing.T) { var recs []cid.Cid head := ncid + var lastRec string + for loop := 0; loop < 50; loop++ { - for i := 0; i < 100; i++ { + for i := 0; i < 20; i++ { ds, err := cs.NewDeltaSession(ctx, 1, &rev) if err != nil { t.Fatal(err) @@ -188,16 +192,23 @@ func TestRepeatedCompactions(t *testing.T) { if err != nil { t.Fatal(err) } - - rc, _, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ - Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), - }) - if err != nil { - t.Fatal(err) + if i%4 == 3 { + if err := rr.DeleteRecord(ctx, lastRec); err != nil { + t.Fatal(err) + } + recs = recs[:len(recs)-1] + } else { + rc, tid, err := rr.CreateRecord(ctx, "app.bsky.feed.post", &appbsky.FeedPost{ + Text: fmt.Sprintf("hey look its a tweet %d", time.Now().UnixNano()), + }) + if err != nil { + t.Fatal(err) + } + + recs = append(recs, rc) + lastRec = "app.bsky.feed.post/" + tid } - recs = append(recs, rc) - kmgr := &util.FakeKeyManager{} nroot, nrev, err := rr.Commit(ctx, kmgr.SignForUser) if err != nil { @@ -228,21 +239,21 @@ func TestRepeatedCompactions(t *testing.T) { if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { t.Fatal(err) } - checkRepo(t, buf, recs) + checkRepo(t, cs, buf, recs) } buf := new(bytes.Buffer) if err := cs.ReadUserCar(ctx, 1, "", true, buf); err != nil { t.Fatal(err) } - checkRepo(t, buf, recs) + checkRepo(t, cs, buf, recs) } -func checkRepo(t *testing.T, r io.Reader, expRecs []cid.Cid) { +func checkRepo(t *testing.T, cs *CarStore, r io.Reader, expRecs []cid.Cid) { t.Helper() rep, err := repo.ReadRepoFromCar(context.TODO(), r) if err != nil { - t.Fatal(err) + t.Fatal("Reading repo: ", err) } set := make(map[cid.Cid]bool) @@ -259,7 +270,23 @@ func checkRepo(t *testing.T, r io.Reader, expRecs []cid.Cid) { return nil }); err != nil { - t.Fatal(err) + var ierr ipld.ErrNotFound + if errors.As(err, &ierr) { + fmt.Println("matched error") + bs, err := cs.ReadOnlySession(1) + if err != nil { + fmt.Println("couldnt read session: ", err) + } + + blk, err := bs.Get(context.TODO(), ierr.Cid) + if err != nil { + fmt.Println("also failed the local get: ", err) + } else { + fmt.Println("LOCAL GET SUCCESS", len(blk.RawData())) + } + } + + t.Fatal("walking repo: ", err) } if len(set) > 0 { diff --git a/cmd/gosky/debug.go b/cmd/gosky/debug.go index 382b5aab..3822e392 100644 --- a/cmd/gosky/debug.go +++ b/cmd/gosky/debug.go @@ -775,7 +775,7 @@ var debugGetRepoCmd = &cli.Command{ if err := rep.ForEach(ctx, "", func(k string, v cid.Cid) error { rec, err := rep.Blockstore().Get(ctx, v) if err != nil { - return err + return fmt.Errorf("getting record %q: %w", k, err) } count++ diff --git a/mst/mst.go b/mst/mst.go index 47c9b0fd..ee648add 100644 --- a/mst/mst.go +++ b/mst/mst.go @@ -944,26 +944,26 @@ func (mst *MerkleSearchTree) WalkLeavesFrom(ctx context.Context, from string, cb entries, err := mst.getEntries(ctx) if err != nil { - return err + return fmt.Errorf("get entries: %w", err) } if index > 0 { prev := entries[index-1] if !prev.isUndefined() && prev.isTree() { if err := prev.Tree.WalkLeavesFrom(ctx, from, cb); err != nil { - return err + return fmt.Errorf("walk leaves %d: %w", index, err) } } } - for _, e := range entries[index:] { + for i, e := range entries[index:] { if e.isLeaf() { if err := cb(e.Key, e.Val); err != nil { return err } } else { if err := e.Tree.WalkLeavesFrom(ctx, from, cb); err != nil { - return err + return fmt.Errorf("walk leaves from (%d): %w", i, err) } } } -- 2.51.2 From 98242a4dfae69057b7699d455f3aa02fb20d8449 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Sun, 1 Oct 2023 16:37:52 +0000 Subject: [PATCH 083/192] Use custom metrics middleware for Palomar to avoid 404DOSing --- search/metrics.go | 93 +++++++++++++++++++++++++++++++++++++++++++++++ search/server.go | 6 +-- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/search/metrics.go b/search/metrics.go index 829c9bc1..bd53581e 100644 --- a/search/metrics.go +++ b/search/metrics.go @@ -1,6 +1,12 @@ package search import ( + "errors" + "net/http" + "strconv" + "time" + + "github.com/labstack/echo/v4" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" ) @@ -49,3 +55,90 @@ var currentSeq = promauto.NewGauge(prometheus.GaugeOpts{ Name: "search_current_seq", Help: "Current sequence number", }) + +var reqSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_request_size_bytes", + Help: "A histogram of request sizes for requests.", + Buckets: prometheus.ExponentialBuckets(100, 10, 8), +}, []string{"code", "method", "path"}) + +var reqDur = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_request_duration_seconds", + Help: "A histogram of latencies for requests.", + Buckets: prometheus.ExponentialBuckets(0.0001, 2, 18), +}, []string{"code", "method", "path"}) + +var reqCnt = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "A counter for requests to the wrapped handler.", +}, []string{"code", "method", "path"}) + +var resSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "http_response_size_bytes", + Help: "A histogram of response sizes for requests.", + Buckets: prometheus.ExponentialBuckets(100, 10, 8), +}, []string{"code", "method", "path"}) + +// MetricsMiddleware defines handler function for metrics middleware +func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + path := c.Path() + if path == "/metrics" || path == "/_health" { + return next(c) + } + + start := time.Now() + requestSize := computeApproximateRequestSize(c.Request()) + + err := next(c) + + status := c.Response().Status + if err != nil { + var httpError *echo.HTTPError + if errors.As(err, &httpError) { + status = httpError.Code + } + if status == 0 || status == http.StatusOK { + status = http.StatusInternalServerError + } + } + + elapsed := float64(time.Since(start)) / float64(time.Second) + + statusStr := strconv.Itoa(status) + method := c.Request().Method + + responseSize := float64(c.Response().Size) + + reqDur.WithLabelValues(statusStr, method, path).Observe(elapsed) + reqCnt.WithLabelValues(statusStr, method, path).Inc() + reqSz.WithLabelValues(statusStr, method, path).Observe(float64(requestSize)) + resSz.WithLabelValues(statusStr, method, path).Observe(responseSize) + + return err + } +} + +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s = len(r.URL.Path) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} diff --git a/search/server.go b/search/server.go index 7a80b27d..82dbccd2 100644 --- a/search/server.go +++ b/search/server.go @@ -13,8 +13,8 @@ import ( "github.com/bluesky-social/indigo/backfill" "github.com/bluesky-social/indigo/util/version" "github.com/bluesky-social/indigo/xrpc" + "github.com/prometheus/client_golang/prometheus/promhttp" - "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" es "github.com/opensearch-project/opensearch-go/v2" @@ -183,7 +183,7 @@ func (s *Server) RunAPI(listen string) error { e.HideBanner = true e.Use(slogecho.New(s.logger)) e.Use(middleware.Recover()) - e.Use(echoprometheus.NewMiddleware("palomar")) + e.Use(MetricsMiddleware) e.Use(middleware.BodyLimit("64M")) e.HTTPErrorHandler = func(err error, ctx echo.Context) { @@ -197,7 +197,7 @@ func (s *Server) RunAPI(listen string) error { e.Use(middleware.CORS()) e.GET("/_health", s.handleHealthCheck) - e.GET("/metrics", echoprometheus.NewHandler()) + e.GET("/metrics", echo.WrapHandler(promhttp.Handler())) e.GET("/xrpc/app.bsky.unspecced.searchPostsSkeleton", s.handleSearchPostsSkeleton) e.GET("/xrpc/app.bsky.unspecced.searchActorsSkeleton", s.handleSearchActorsSkeleton) e.GET("/xrpc/app.bsky.unspecced.indexRepos", s.handleIndexRepos) -- 2.51.2 From a33e87b4859b353e072cab4b0aba52e4383ee109 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Sun, 1 Oct 2023 13:53:21 -0700 Subject: [PATCH 084/192] have compaction endpoints return more information --- bgs/admin.go | 22 ++++++++++++++++++---- bgs/bgs.go | 31 ++++++++++++++++++++++++++----- carstore/bs.go | 10 ++++++---- 3 files changed, 50 insertions(+), 13 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index 13504125..4044c9c1 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -426,13 +426,27 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { ctx, span := otel.Tracer("bgs").Start(context.Background(), "adminCompactAllRepos") defer span.End() - if err := bgs.runRepoCompaction(ctx); err != nil { + var dry bool + if strings.ToLower(e.QueryParam("dry")) == "true" { + dry = true + } + + lim := 50 + if limstr := e.QueryParam("limit"); limstr != "" { + v, err := strconv.Atoi(limstr) + if err != nil { + return err + } + + lim = v + } + + stats, err := bgs.runRepoCompaction(ctx, lim, dry) + if err != nil { return fmt.Errorf("compaction run failed: %w", err) } - return e.JSON(200, map[string]any{ - "success": "true", - }) + return e.JSON(200, stats) } func (bgs *BGS) handleAdminPostResyncPDS(e echo.Context) error { diff --git a/bgs/bgs.go b/bgs/bgs.go index 1c359fa2..e3d87bdc 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1127,23 +1127,44 @@ func (bgs *BGS) ReverseTakedown(ctx context.Context, did string) error { return nil } -func (bgs *BGS) runRepoCompaction(ctx context.Context) error { +type compactionStats struct { + Completed map[models.Uid]*carstore.CompactionStats + Targets []carstore.CompactionTarget +} + +func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*compactionStats, error) { ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction") defer span.End() - repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx) + repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50) if err != nil { - return fmt.Errorf("failed to get repos to compact: %w", err) + return nil, fmt.Errorf("failed to get repos to compact: %w", err) + } + + if lim > 0 && len(repos) > lim { + repos = repos[:lim] } + if dry { + return &compactionStats{ + Targets: repos, + }, nil + } + + results := make(map[models.Uid]*carstore.CompactionStats) for _, r := range repos { - if _, err := bgs.repoman.CarStore().CompactUserShards(ctx, r.Usr); err != nil { + st, err := bgs.repoman.CarStore().CompactUserShards(ctx, r.Usr) + if err != nil { log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) continue } + results[r.Usr] = st } - return nil + return &compactionStats{ + Targets: repos, + Completed: results, + }, nil } type repoHead struct { diff --git a/carstore/bs.go b/carstore/bs.go index e646df8f..4dd05735 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -727,7 +727,7 @@ func createBlockRefs(ctx context.Context, tx *gorm.DB, brefs []map[string]any) e ctx, span := otel.Tracer("carstore").Start(ctx, "createBlockRefs") defer span.End() - if err := createInBatches(ctx, tx, brefs, 100); err != nil { + if err := createInBatches(ctx, tx, brefs, 1000); err != nil { return err } @@ -1150,12 +1150,12 @@ type CompactionTarget struct { NumShards int } -func (cs *CarStore) GetCompactionTargets(ctx context.Context) ([]CompactionTarget, error) { +func (cs *CarStore) GetCompactionTargets(ctx context.Context, shardCount int) ([]CompactionTarget, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "GetCompactionTargets") defer span.End() var targets []CompactionTarget - if err := cs.meta.Raw(`select usr, count(*) as num_shards from car_shards group by usr having count(*) > 50 order by num_shards desc`).Scan(&targets).Error; err != nil { + if err := cs.meta.Raw(`select usr, count(*) as num_shards from car_shards group by usr having count(*) > ? order by num_shards desc`, shardCount).Scan(&targets).Error; err != nil { return nil, err } @@ -1186,6 +1186,7 @@ func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) } type CompactionStats struct { + TotalRefs int `json:"totalRefs"` StartShards int `json:"startShards"` NewShards int `json:"newShards"` SkippedShards int `json:"skippedShards"` @@ -1326,6 +1327,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co stats := &CompactionStats{ StartShards: len(shards), + TotalRefs: len(brefs), } removedShards := make(map[uint]bool) @@ -1397,7 +1399,7 @@ func (cs *CarStore) deleteStaleRefs(ctx context.Context, brefs []blockRef, stale } } - chunkSize := 100 + chunkSize := 500 for i := 0; i < len(staleToDelete); i += chunkSize { sl := staleToDelete[i:] if len(sl) > chunkSize { -- 2.51.2 From bf081a3f88917931f09d58f2c3238c220f6fa831 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 2 Oct 2023 10:56:40 -0700 Subject: [PATCH 085/192] improvement to compact-all command --- cmd/gosky/bgs.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/cmd/gosky/bgs.go b/cmd/gosky/bgs.go index dfbddb68..6764a304 100644 --- a/cmd/gosky/bgs.go +++ b/cmd/gosky/bgs.go @@ -347,10 +347,29 @@ var bgsCompactRepo = &cli.Command{ var bgsCompactAll = &cli.Command{ Name: "compact-all", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "dry", + }, + &cli.IntFlag{ + Name: "limit", + }, + }, Action: func(cctx *cli.Context) error { - url := cctx.String("bgs") + "/admin/repo/compactAll" + uu, err := url.Parse(cctx.String("bgs") + "/admin/repo/compactAll") + if err != nil { + return err + } - req, err := http.NewRequest("POST", url, nil) + if cctx.Bool("dry") { + uu.Query().Add("dry", "true") + } + + if cctx.IsSet("limit") { + uu.Query().Add("limit", fmt.Sprint(cctx.Int("limit"))) + } + + req, err := http.NewRequest("POST", uu.String(), nil) if err != nil { return err } -- 2.51.2 From 1d320433ea790323ffc05164fd74d9235cd19925 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 25 Sep 2023 14:37:59 -0700 Subject: [PATCH 086/192] implement repo tombstone handler --- bgs/bgs.go | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index e3d87bdc..019be941 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -453,7 +453,8 @@ type User struct { // TakenDown is set to true if the user in question has been taken down. // A user in this state will have all future events related to it dropped // and no data about this user will be served. - TakenDown bool + TakenDown bool + Tombstoned bool } type addTargetBody struct { @@ -860,12 +861,52 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event return err } + return nil + case env.RepoTombstone != nil: + if err := bgs.handleRepoTombstone(ctx, host, env.RepoTombstone); err != nil { + return err + } + return nil default: return fmt.Errorf("invalid fed event") } } +func (bgs *BGS) handleRepoTombstone(ctx context.Context, pds *models.PDS, evt *atproto.SyncSubscribeRepos_Tombstone) error { + u, err := bgs.lookupUserByDid(ctx, evt.Did) + if err != nil { + return err + } + + if u.PDS != pds.ID { + return fmt.Errorf("unauthoritative tombstone event from %s for %s", pds.Host, evt.Did) + } + + if err := bgs.db.Model(&User{}).Where("id = ?", u.ID).UpdateColumns(map[string]any{ + "tombstoned": true, + "handle": nil, + }).Error; err != nil { + return err + } + + if err := bgs.db.Model(&models.ActorInfo{}).Where("uid = ?", u.ID).UpdateColumns(map[string]any{ + "handle": nil, + }).Error; err != nil { + return err + } + + // delete data from carstore + if err := bgs.repoman.TakeDownRepo(ctx, u.ID); err != nil { + // don't let a failure here prevent us from propagating this event + log.Errorf("failed to delete user data from carstore: %s", err) + } + + return bgs.events.AddEvent(ctx, &events.XRPCStreamEvent{ + RepoTombstone: evt, + }) +} + func (s *BGS) syncUserBlobs(ctx context.Context, pds *models.PDS, user models.Uid, blobs []string) error { if s.blobs == nil { log.Debugf("blob syncing disabled") -- 2.51.2 From 735db31d5c508468d01acb96312ed79c08d0fda1 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 25 Sep 2023 15:20:39 -0700 Subject: [PATCH 087/192] respect user flags in sync endpoints --- bgs/handlers.go | 41 +++++++++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 8 deletions(-) diff --git a/bgs/handlers.go b/bgs/handlers.go index dc282717..133cc052 100644 --- a/bgs/handlers.go +++ b/bgs/handlers.go @@ -21,7 +21,7 @@ import ( ) func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection string, commit string, did string, rkey string) (io.Reader, error) { - u, err := s.Index.LookupUserByDid(ctx, did) + u, err := s.lookupUserByDid(ctx, did) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") @@ -29,6 +29,14 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") } + if u.Tombstoned { + return nil, fmt.Errorf("account was deleted") + } + + if u.TakenDown { + return nil, fmt.Errorf("account was taken down") + } + reqCid := cid.Undef if commit != "" { reqCid, err = cid.Decode(commit) @@ -37,7 +45,7 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri } } - _, record, err := s.repoman.GetRecord(ctx, u.Uid, collection, rkey, reqCid) + _, record, err := s.repoman.GetRecord(ctx, u.ID, collection, rkey, reqCid) if err != nil { return nil, fmt.Errorf("failed to get record: %w", err) } @@ -52,7 +60,7 @@ func (s *BGS) handleComAtprotoSyncGetRecord(ctx context.Context, collection stri } func (s *BGS) handleComAtprotoSyncGetRepo(ctx context.Context, did string, since string) (io.Reader, error) { - u, err := s.Index.LookupUserByDid(ctx, did) + u, err := s.lookupUserByDid(ctx, did) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") @@ -60,9 +68,17 @@ func (s *BGS) handleComAtprotoSyncGetRepo(ctx context.Context, did string, since return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") } + if u.Tombstoned { + return nil, fmt.Errorf("account was deleted") + } + + if u.TakenDown { + return nil, fmt.Errorf("account was taken down") + } + // TODO: stream the response buf := new(bytes.Buffer) - if err := s.repoman.ReadRepo(ctx, u.Uid, since, buf); err != nil { + if err := s.repoman.ReadRepo(ctx, u.ID, since, buf); err != nil { return nil, fmt.Errorf("failed to read repo: %w", err) } @@ -158,7 +174,7 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, } users := []User{} - if err := s.db.Model(&User{}).Where("id > ?", c).Order("id").Limit(limit).Find(&users).Error; err != nil { + if err := s.db.Model(&User{}).Where("id > ? AND NOT tombstoned AND NOT taken_down", c).Order("id").Limit(limit).Find(&users).Error; err != nil { if err == gorm.ErrRecordNotFound { return &comatprototypes.SyncListRepos_Output{}, nil } @@ -175,6 +191,7 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, for i := range users { user := users[i] + root, err := s.repoman.GetRepoRoot(ctx, user.ID) if err != nil { return nil, fmt.Errorf("failed to get repo root for (%s): %w", user.Did, err) @@ -194,7 +211,7 @@ func (s *BGS) handleComAtprotoSyncListRepos(ctx context.Context, cursor string, } func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context, did string) (*comatprototypes.SyncGetLatestCommit_Output, error) { - u, err := s.Index.LookupUserByDid(ctx, did) + u, err := s.lookupUserByDid(ctx, did) if err != nil { if errors.Is(err, gorm.ErrRecordNotFound) { return nil, echo.NewHTTPError(http.StatusNotFound, "user not found") @@ -202,12 +219,20 @@ func (s *BGS) handleComAtprotoSyncGetLatestCommit(ctx context.Context, did strin return nil, echo.NewHTTPError(http.StatusInternalServerError, "failed to lookup user") } - root, err := s.repoman.GetRepoRoot(ctx, u.Uid) + if u.Tombstoned { + return nil, fmt.Errorf("account was deleted") + } + + if u.TakenDown { + return nil, fmt.Errorf("account was taken down") + } + + root, err := s.repoman.GetRepoRoot(ctx, u.ID) if err != nil { return nil, fmt.Errorf("failed to get repo root: %w", err) } - rev, err := s.repoman.GetRepoRev(ctx, u.Uid) + rev, err := s.repoman.GetRepoRev(ctx, u.ID) if err != nil { return nil, fmt.Errorf("failed to get repo rev: %w", err) } -- 2.51.2 From b60c71ea3359f46702c9e3839cfedcfefffe2050 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 28 Sep 2023 11:04:43 -0700 Subject: [PATCH 088/192] handle un-tombstoning --- bgs/bgs.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index 019be941..5b22595c 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -777,6 +777,33 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event return fmt.Errorf("rebase was true in event seq:%d,host:%s", evt.Seq, host.Host) } + if host.ID != u.PDS { + log.Infow("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host) + subj, err := bgs.createExternalUser(ctx, evt.Repo) + if err != nil { + return err + } + + if subj.PDS != host.ID { + return fmt.Errorf("event from non-authoritative pds") + } + } + + if u.Tombstoned { + // we've checked the authority of the users PDS, so reinstate the account + if err := bgs.db.Model(&User{}).Where("id = ?", u.ID).UpdateColumn("tombstoned", false).Error; err != nil { + return fmt.Errorf("failed to un-tombstone a user: %w", err) + } + + ai, err := bgs.Index.LookupUser(ctx, u.ID) + if err != nil { + return fmt.Errorf("failed to look up user (tombstone recover): %w", err) + } + + // Now a simple re-crawl should suffice to bring the user back online + return bgs.Index.Crawler.AddToCatchupQueue(ctx, host, ai, evt) + } + // skip the fast path for rebases or if the user is already in the slow path if bgs.Index.Crawler.RepoInSlowPath(ctx, host, u.ID) { rebasesCounter.WithLabelValues(host.Host).Add(1) @@ -1050,6 +1077,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor return nil, fmt.Errorf("failed to update users pds: %w", err) } + exu.PDS = peering.ID } if exu.Handle.String != handle { -- 2.51.2 From 6253e67ec3ce00a754253916058f66edd13a50ec Mon Sep 17 00:00:00 2001 From: Ryan Barrett Date: Mon, 2 Oct 2023 12:54:37 -0700 Subject: [PATCH 089/192] BGS: allow non-SSL BGS with SSL PDS seems like this should hopefully be allowed for testing, and the main combination we want to prevent is SSL (ie prod) BGS with non-SSL PDS...? --- bgs/bgs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 5b22595c..245afe40 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1026,7 +1026,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor peering.Host = durl.Host peering.SSL = (durl.Scheme == "https") - if s.ssl != peering.SSL { + if s.ssl && !peering.SSL { return nil, fmt.Errorf("did references non-ssl PDS, this is disallowed in prod: %q %q", did, svc.ServiceEndpoint) } -- 2.51.2 From 11b4a36a0ec9881a0c559b053d0d20565ec7031d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 2 Oct 2023 16:28:08 -0700 Subject: [PATCH 090/192] make fewer traces --- carstore/bs.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index 4dd05735..a1afebd0 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -454,9 +454,6 @@ func (cs *CarStore) writeBlockFromShard(ctx context.Context, sh *CarShard, w io. } func (cs *CarStore) iterateShardBlocks(ctx context.Context, sh *CarShard, cb func(blk blockformat.Block) error) error { - ctx, span := otel.Tracer("carstore").Start(ctx, "iterateShardBlocks") - defer span.End() - fi, err := os.Open(sh.Path) if err != nil { return err @@ -1418,6 +1415,8 @@ func (cs *CarStore) compactBucket(ctx context.Context, user models.Uid, b *compB ctx, span := otel.Tracer("carstore").Start(ctx, "compactBucket") defer span.End() + span.SetAttributes(attribute.Int("shards", len(b.shards))) + last := b.shards[len(b.shards)-1] lastsh := shardsById[last.ID] fi, path, err := cs.openNewCompactedShardFile(ctx, user, last.Seq) -- 2.51.2 From 80e4e45b4424970032a5bdc245216957199f39fd Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 2 Oct 2023 17:18:50 -0700 Subject: [PATCH 091/192] fixup threshold bounds check --- carstore/bs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carstore/bs.go b/carstore/bs.go index a1afebd0..497fd337 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1299,7 +1299,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } thresholdForPosition := func(i int) int { - if i > len(threshs) { + if i >= len(threshs) { return lowBound } return threshs[i] -- 2.51.2 From 89591d5960fe44bd09d26706428c2d573c13c928 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 16:22:36 +0000 Subject: [PATCH 092/192] Fix MST key validation to allow single-character rkeys --- mst/mst_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mst/mst_util.go b/mst/mst_util.go index 22af517a..7fb004eb 100644 --- a/mst/mst_util.go +++ b/mst/mst_util.go @@ -226,7 +226,7 @@ func isValidMstKey(s string) bool { return false } a, b, _ := strings.Cut(s, "/") - return len(b) > 1 && + return len(b) > 0 && keyHasAllValidChars(a) && keyHasAllValidChars(b) } -- 2.51.2 From 76449d94c8c229c7aed790c57445233799cab070 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 16:28:11 +0000 Subject: [PATCH 093/192] Don't complain about feed types we know about --- indexer/indexer.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/indexer/indexer.go b/indexer/indexer.go index 31d6745c..f47fc05a 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -276,6 +276,12 @@ func (ix *Indexer) crawlRecordReferences(ctx context.Context, op *repomgr.RepoOp return nil case *bsky.ActorProfile: return nil + case *bsky.GraphList: + return nil + case *bsky.GraphListitem: + return nil + case *bsky.FeedGenerator: + return nil default: log.Warnf("unrecognized record type: %T", op.Record) return nil -- 2.51.2 From 34f85a8a586df1e111f26e36d92b394170767b8a Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 16:51:53 +0000 Subject: [PATCH 094/192] Create an index on active users to make paging through them more efficient --- bgs/bgs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 245afe40..1888112d 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -441,7 +441,7 @@ func (bgs *BGS) checkAdminAuth(next echo.HandlerFunc) echo.HandlerFunc { } type User struct { - ID models.Uid `gorm:"primarykey"` + ID models.Uid `gorm:"primarykey;index:idx_user_id_active,where:taken_down = false AND tombstoned = false"` CreatedAt time.Time UpdatedAt time.Time DeletedAt gorm.DeletedAt `gorm:"index"` -- 2.51.2 From 0e52acfdf30fd83c67f48bb594beecc393551615 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 17:37:39 +0000 Subject: [PATCH 095/192] Add debug command to compare the same repo across two hosts --- cmd/gosky/debug.go | 142 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/cmd/gosky/debug.go b/cmd/gosky/debug.go index 3822e392..bdd71451 100644 --- a/cmd/gosky/debug.go +++ b/cmd/gosky/debug.go @@ -11,6 +11,7 @@ import ( "os/signal" "strconv" "strings" + "sync" "syscall" "github.com/bluesky-social/indigo/api/atproto" @@ -28,6 +29,7 @@ import ( "github.com/gorilla/websocket" "github.com/ipfs/go-cid" + "github.com/ipfs/go-libipfs/blocks" "github.com/ipld/go-car/v2" cli "github.com/urfave/cli/v2" ) @@ -42,6 +44,7 @@ var debugCmd = &cli.Command{ debugFeedViewCmd, compareStreamsCmd, debugGetRepoCmd, + debugCompareReposCmd, }, } @@ -789,3 +792,142 @@ var debugGetRepoCmd = &cli.Command{ return nil }, } + +var debugCompareReposCmd = &cli.Command{ + Name: "compare-repos", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "host-1", + Usage: "method, hostname, and port of PDS instance", + Value: "https://bsky.social", + }, + &cli.StringFlag{ + Name: "host-2", + Usage: "method, hostname, and port of PDS instance", + Value: "https://bgs.bsky.social", + }, + }, + ArgsUsage: ``, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + + wg := sync.WaitGroup{} + wg.Add(2) + + xrpc1 := xrpc.Client{ + Host: cctx.String("host-1"), + } + + xrpc2 := xrpc.Client{ + Host: cctx.String("host-2"), + } + + var rep1 *repo.Repo + go func() { + defer wg.Done() + logger := log.With("host", cctx.String("host-1")) + repo1bytes, err := comatproto.SyncGetRepo(ctx, &xrpc1, cctx.Args().First(), "") + if err != nil { + logger.Fatalf("getting repo: %s", err) + return + } + + rep1, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo1bytes)) + if err != nil { + logger.Fatalf("reading repo: %s", err) + return + } + }() + + var rep2 *repo.Repo + go func() { + defer wg.Done() + logger := log.With("host", cctx.String("host-2")) + repo2bytes, err := comatproto.SyncGetRepo(ctx, &xrpc2, cctx.Args().First(), "") + if err != nil { + logger.Fatalf("getting repo: %s", err) + return + } + + rep2, err = repo.ReadRepoFromCar(ctx, bytes.NewReader(repo2bytes)) + if err != nil { + logger.Fatalf("reading repo: %s", err) + return + } + }() + + wg.Wait() + + cids1 := []cid.Cid{} + blocks1 := []blocks.Block{} + + fmt.Println("Host 1 Results") + fmt.Println("Rev: ", rep1.SignedCommit().Rev) + var count int + if err := rep1.ForEach(ctx, "", func(k string, v cid.Cid) error { + cids1 = append(cids1, v) + rec, err := rep1.Blockstore().Get(ctx, v) + if err != nil { + return fmt.Errorf("getting record %q: %w", k, err) + } + blocks1 = append(blocks1, rec) + + count++ + _ = rec + return nil + }); err != nil { + return err + } + fmt.Printf("scanned %d records\n", count) + + cids2 := []cid.Cid{} + blocks2 := []blocks.Block{} + + fmt.Println("\nHost 2 Results") + fmt.Println("Rev: ", rep2.SignedCommit().Rev) + count = 0 + if err := rep2.ForEach(ctx, "", func(k string, v cid.Cid) error { + cids2 = append(cids2, v) + rec, err := rep2.Blockstore().Get(ctx, v) + if err != nil { + return fmt.Errorf("getting record %q: %w", k, err) + } + blocks2 = append(blocks2, rec) + + count++ + _ = rec + return nil + }); err != nil { + return err + } + fmt.Printf("scanned %d records\n", count) + + fmt.Println("\nComparing CIDs") + hasBadCid := false + for i, c1 := range cids1 { + if c1 != cids2[i] { + fmt.Printf("CID mismatch at index %d: %s != %s\n", i, c1, cids2[i]) + hasBadCid = true + } + } + + if !hasBadCid { + fmt.Println("All CIDs match!") + } + + fmt.Println("Comparing blocks") + hasBadBlock := false + for i, b1 := range blocks1 { + if !bytes.Equal(b1.RawData(), blocks2[i].RawData()) { + fmt.Printf("Block mismatch at index %d Host 1 Cid (%s) Host 2 Cid (%s)\n", i, b1.Cid().String(), blocks2[i].Cid().String()) + hasBadBlock = true + } + } + + if !hasBadBlock { + fmt.Println("All blocks match!") + } + + return nil + }, +} -- 2.51.2 From 63ec6ad7e0e480197420e48ec559ce42460ec308 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 17:38:32 +0000 Subject: [PATCH 096/192] Return error for bad status code --- cmd/gosky/debug.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/gosky/debug.go b/cmd/gosky/debug.go index bdd71451..c8845ab4 100644 --- a/cmd/gosky/debug.go +++ b/cmd/gosky/debug.go @@ -928,6 +928,10 @@ var debugCompareReposCmd = &cli.Command{ fmt.Println("All blocks match!") } + if hasBadBlock || hasBadCid { + return fmt.Errorf("mismatched blocks or cids") + } + return nil }, } -- 2.51.2 From bdc613431344434f034f192a8cef3bde743441d9 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 17:43:23 +0000 Subject: [PATCH 097/192] Remove automaxprocs from gosky --- cmd/gosky/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/gosky/main.go b/cmd/gosky/main.go index c85afe27..87f18bed 100644 --- a/cmd/gosky/main.go +++ b/cmd/gosky/main.go @@ -38,7 +38,6 @@ import ( "github.com/ipld/go-car" _ "github.com/joho/godotenv/autoload" - _ "go.uber.org/automaxprocs" logging "github.com/ipfs/go-log" "github.com/polydawn/refmt/cbor" -- 2.51.2 From d56fdbf42eadcfcef497df2b1eacdddd5707cce2 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 3 Oct 2023 17:46:24 +0000 Subject: [PATCH 098/192] Extend HTTP timeout --- cmd/gosky/debug.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/gosky/debug.go b/cmd/gosky/debug.go index c8845ab4..7ecdfff1 100644 --- a/cmd/gosky/debug.go +++ b/cmd/gosky/debug.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/bluesky-social/indigo/api/atproto" comatproto "github.com/bluesky-social/indigo/api/atproto" @@ -816,10 +817,16 @@ var debugCompareReposCmd = &cli.Command{ xrpc1 := xrpc.Client{ Host: cctx.String("host-1"), + Client: &http.Client{ + Timeout: 15 * time.Minute, + }, } xrpc2 := xrpc.Client{ Host: cctx.String("host-2"), + Client: &http.Client{ + Timeout: 15 * time.Minute, + }, } var rep1 *repo.Repo -- 2.51.2 From 5339281d40f583cf7a5212aeb9944ddd660307aa Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 3 Oct 2023 11:36:21 -0700 Subject: [PATCH 099/192] more trace attributes and some tweaks --- bgs/bgs.go | 7 ++++++- bgs/metrics.go | 6 ++++++ carstore/bs.go | 31 ++++++++++++++++++++++++++++--- cmd/gosky/main.go | 2 +- repomgr/repomgr.go | 3 +++ 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 245afe40..c3501ff4 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -744,6 +744,11 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event ctx, span := otel.Tracer("bgs").Start(ctx, "handleFedEvent") defer span.End() + start := time.Now() + defer func() { + eventsHandleDuration.WithLabelValues(host.Host).Observe(time.Since(start).Seconds()) + }() + eventsReceivedCounter.WithLabelValues(host.Host).Add(1) switch { @@ -1222,7 +1227,7 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp results := make(map[models.Uid]*carstore.CompactionStats) for _, r := range repos { - st, err := bgs.repoman.CarStore().CompactUserShards(ctx, r.Usr) + st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr) if err != nil { log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) continue diff --git a/bgs/metrics.go b/bgs/metrics.go index b45f06b0..97f444c4 100644 --- a/bgs/metrics.go +++ b/bgs/metrics.go @@ -16,6 +16,12 @@ var eventsReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Help: "The total number of events received", }, []string{"pds"}) +var eventsHandleDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "events_handle_duration", + Help: "A histogram of handleFedEvent latencies", + Buckets: prometheus.ExponentialBuckets(0.001, 2, 15), +}, []string{"pds"}) + var repoCommitsReceivedCounter = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "repo_commits_received_counter", Help: "The total number of events received", diff --git a/carstore/bs.go b/carstore/bs.go index 497fd337..2adb7e3c 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -164,13 +164,30 @@ func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, erro } } +const prefetchThreshold = 512 << 10 + func (uv *userView) prefetchRead(ctx context.Context, k cid.Cid, path string, offset int64) (blockformat.Block, error) { + ctx, span := otel.Tracer("carstore").Start(ctx, "getLastShard") + defer span.End() + fi, err := os.Open(path) if err != nil { return nil, err } defer fi.Close() + st, err := fi.Stat() + if err != nil { + return nil, fmt.Errorf("stat file for prefetch: %w", err) + } + + span.SetAttributes(attribute.Int64("shard_size", st.Size())) + + if st.Size() > prefetchThreshold { + span.SetAttributes(attribute.Bool("no_prefetch", true)) + return doBlockRead(fi, k, offset) + } + cr, err := car.NewCarReader(fi) if err != nil { return nil, err @@ -203,6 +220,10 @@ func (uv *userView) singleRead(ctx context.Context, k cid.Cid, path string, offs } defer fi.Close() + return doBlockRead(fi, k, offset) +} + +func doBlockRead(fi *os.File, k cid.Cid, offset int64) (blockformat.Block, error) { seeked, err := fi.Seek(offset, io.SeekStart) if err != nil { return nil, err @@ -724,7 +745,7 @@ func createBlockRefs(ctx context.Context, tx *gorm.DB, brefs []map[string]any) e ctx, span := otel.Tracer("carstore").Start(ctx, "createBlockRefs") defer span.End() - if err := createInBatches(ctx, tx, brefs, 1000); err != nil { + if err := createInBatches(ctx, tx, brefs, 2000); err != nil { return err } @@ -1008,7 +1029,7 @@ func (cs *CarStore) deleteShards(ctx context.Context, shs []*CarShard) error { return outErr } - chunkSize := 100 + chunkSize := 10000 for i := 0; i < len(shs); i += chunkSize { sl := shs[i:] if len(sl) > chunkSize { @@ -1163,6 +1184,8 @@ func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) ctx, span := otel.Tracer("carstore").Start(ctx, "getBlockRefsForShards") defer span.End() + span.SetAttributes(attribute.Int("shards", len(shardIds))) + chunkSize := 10000 out := make([]blockRef, 0, len(shardIds)) for i := 0; i < len(shardIds); i += chunkSize { @@ -1179,6 +1202,8 @@ func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) out = append(out, brefs...) } + span.SetAttributes(attribute.Int("refs", len(out))) + return out, nil } @@ -1396,7 +1421,7 @@ func (cs *CarStore) deleteStaleRefs(ctx context.Context, brefs []blockRef, stale } } - chunkSize := 500 + chunkSize := 10000 for i := 0; i < len(staleToDelete); i += chunkSize { sl := staleToDelete[i:] if len(sl) > chunkSize { diff --git a/cmd/gosky/main.go b/cmd/gosky/main.go index 87f18bed..30ba23ed 100644 --- a/cmd/gosky/main.go +++ b/cmd/gosky/main.go @@ -199,7 +199,7 @@ var postCmd = &cli.Command{ Repo: auth.Did, Record: &lexutil.LexiconTypeDecoder{&appbsky.FeedPost{ Text: text, - CreatedAt: time.Now().Format(util.ISO8601), + CreatedAt: time.Now().UTC().Format(util.ISO8601), }}, }) if err != nil { diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index e9ffece4..1a5d07d5 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -25,6 +25,7 @@ import ( "github.com/ipld/go-car" cbg "github.com/whyrusleeping/cbor-gen" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "gorm.io/gorm" ) @@ -490,6 +491,8 @@ func (rm *RepoManager) HandleExternalUserEvent(ctx context.Context, pdsid uint, ctx, span := otel.Tracer("repoman").Start(ctx, "HandleExternalUserEvent") defer span.End() + span.SetAttributes(attribute.Int64("uid", int64(uid))) + log.Infow("HandleExternalUserEvent", "pds", pdsid, "uid", uid, "since", since, "nrev", nrev) unlock := rm.lockUser(ctx, uid) -- 2.51.2 From 1bc437186cc8a5dea929ebcdc788279ea19d7b80 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 03:11:17 -0700 Subject: [PATCH 100/192] identity: add fallback DNS servers --- atproto/identity/base_directory.go | 2 ++ atproto/identity/handle.go | 46 +++++++++++++++++++++++++++++- atproto/identity/live_test.go | 20 +++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/atproto/identity/base_directory.go b/atproto/identity/base_directory.go index 15cec3e2..d53a0bb0 100644 --- a/atproto/identity/base_directory.go +++ b/atproto/identity/base_directory.go @@ -26,6 +26,8 @@ type BaseDirectory struct { TryAuthoritativeDNS bool // set of handle domain suffixes for for which DNS handle resolution will be skipped SkipDNSDomainSuffixes []string + // set of fallback DNS servers (eg, domain registrars) to try as a fallback. each entry should be "ip:port", eg "8.8.8.8:53" + FallbackDNSServers []string } var _ Directory = (*BaseDirectory)(nil) diff --git a/atproto/identity/handle.go b/atproto/identity/handle.go index 41bb473d..dfa3619b 100644 --- a/atproto/identity/handle.go +++ b/atproto/identity/handle.go @@ -89,6 +89,43 @@ func (d *BaseDirectory) ResolveHandleDNSAuthoritative(ctx context.Context, handl return parseTXTResp(res) } +// variant of ResolveHandleDNS which uses any configured fallback DNS servers +func (d *BaseDirectory) ResolveHandleDNSFallback(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { + retErr := fmt.Errorf("no fallback servers configured") + var dnsErr *net.DNSError + for _, ns := range d.FallbackDNSServers { + // create a custom resolver to use the specific nameserver for TXT lookup + resolver := &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, network, address string) (net.Conn, error) { + rd := net.Dialer{ + Timeout: time.Second * 5, + } + return rd.DialContext(ctx, network, ns) + }, + } + res, err := resolver.LookupTXT(ctx, "_atproto."+handle.String()) + // check for NXDOMAIN + if errors.As(err, &dnsErr) { + if dnsErr.IsNotFound { + retErr = ErrHandleNotFound + continue + } + } + if err != nil { + retErr = err + continue + } + ret, err := parseTXTResp(res) + if err != nil { + retErr = err + continue + } + return ret, nil + } + return "", retErr +} + func (d *BaseDirectory) ResolveHandleWellKnown(ctx context.Context, handle syntax.Handle) (syntax.DID, error) { req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("https://%s/.well-known/atproto-did", handle), nil) if err != nil { @@ -138,6 +175,7 @@ func (d *BaseDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) if tryDNS { start := time.Now() triedAuthoritative := false + triedFallback := false did, dnsErr = d.ResolveHandleDNS(ctx, handle) if dnsErr == ErrHandleNotFound && d.TryAuthoritativeDNS { slog.Info("attempting authoritative handle DNS resolution", "handle", handle) @@ -145,8 +183,14 @@ func (d *BaseDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) // try harder with authoritative lookup did, dnsErr = d.ResolveHandleDNSAuthoritative(ctx, handle) } + if dnsErr == ErrHandleNotFound && len(d.FallbackDNSServers) > 0 { + slog.Info("attempting fallback DNS resolution", "handle", handle) + triedFallback = true + // try harder with fallback lookup + did, dnsErr = d.ResolveHandleDNSFallback(ctx, handle) + } elapsed := time.Since(start) - slog.Debug("resolve handle DNS", "handle", handle, "err", dnsErr, "did", did, "authoritative", triedAuthoritative, "duration_ms", elapsed.Milliseconds()) + slog.Debug("resolve handle DNS", "handle", handle, "err", dnsErr, "did", did, "authoritative", triedAuthoritative, "fallback", triedFallback, "duration_ms", elapsed.Milliseconds()) if nil == dnsErr { // if *not* an error return did, nil } diff --git a/atproto/identity/live_test.go b/atproto/identity/live_test.go index 79c16290..59698c30 100644 --- a/atproto/identity/live_test.go +++ b/atproto/identity/live_test.go @@ -115,3 +115,23 @@ func TestCacheCoalesce(t *testing.T) { } wg.Wait() } + +func TestFallbackDNS(t *testing.T) { + assert := assert.New(t) + ctx := context.Background() + handle := syntax.Handle("no-such-record.atproto.com") + dir := BaseDirectory{ + FallbackDNSServers: []string{"1.1.1.1:53", "8.8.8.8:53"}, + } + + // valid DNS server + _, err := dir.LookupHandle(ctx, handle) + assert.Error(err) + assert.Equal(ErrHandleNotFound, err) + + // invalid DNS server syntax + dir.FallbackDNSServers = []string{"_"} + _, err = dir.LookupHandle(ctx, handle) + assert.Error(err) + assert.NotEqual(ErrHandleNotFound, err) +} -- 2.51.2 From c28f90989877a3c4af904c4a741639c0c986930c Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 03:13:12 -0700 Subject: [PATCH 101/192] identity: skip live fallback DNS test by default --- atproto/identity/live_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/atproto/identity/live_test.go b/atproto/identity/live_test.go index 59698c30..c06213d2 100644 --- a/atproto/identity/live_test.go +++ b/atproto/identity/live_test.go @@ -117,6 +117,8 @@ func TestCacheCoalesce(t *testing.T) { } func TestFallbackDNS(t *testing.T) { + t.Skip("TODO: skipping live network test") + assert := assert.New(t) ctx := context.Background() handle := syntax.Handle("no-such-record.atproto.com") -- 2.51.2 From a2b5d72b2de98b6bf4079010cb52dbb70fc23b3e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 3 Oct 2023 16:17:53 -0700 Subject: [PATCH 102/192] respect the contexts a little better --- bgs/bgs.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/bgs/bgs.go b/bgs/bgs.go index c3501ff4..04d475ae 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1227,6 +1227,15 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp results := make(map[models.Uid]*carstore.CompactionStats) for _, r := range repos { + select { + case <-ctx.Done(): + return &compactionStats{ + Targets: repos, + Completed: results, + }, nil + default: + } + st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr) if err != nil { log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) -- 2.51.2 From 7ab055a21ab95409d81da728f3de94013b69f55c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 4 Oct 2023 11:54:44 -0700 Subject: [PATCH 103/192] persisters now handle tombstone events --- carstore/bs.go | 6 ++++++ cmd/gosky/main.go | 14 ++++++++++++++ events/dbpersist.go | 41 +++++++++++++++++++++++++++++++++++++++++ events/diskpersist.go | 24 +++++++++++++++++++++--- 4 files changed, 82 insertions(+), 3 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index 2adb7e3c..56839556 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1228,6 +1228,8 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co return nil, err } + span.SetAttributes(attribute.Int("shards", len(shards))) + var shardIds []uint for _, s := range shards { shardIds = append(shardIds, s.ID) @@ -1243,11 +1245,15 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co return nil, fmt.Errorf("getting block refs failed: %w", err) } + span.SetAttributes(attribute.Int("blockRefs", len(brefs))) + var staleRefs []staleRef if err := cs.meta.WithContext(ctx).Find(&staleRefs, "usr = ?", user).Error; err != nil { return nil, err } + span.SetAttributes(attribute.Int("staleRefs", len(staleRefs))) + stale := make(map[cid.Cid]bool) for _, br := range staleRefs { stale[br.Cid.CID] = true diff --git a/cmd/gosky/main.go b/cmd/gosky/main.go index 30ba23ed..aaec5093 100644 --- a/cmd/gosky/main.go +++ b/cmd/gosky/main.go @@ -1145,6 +1145,20 @@ var readRepoStreamCmd = &cli.Command{ return nil }, + RepoTombstone: func(tomb *comatproto.SyncSubscribeRepos_Tombstone) error { + if jsonfmt { + b, err := json.Marshal(tomb) + if err != nil { + return err + } + fmt.Println(string(b)) + } else { + fmt.Printf("(%d) Tombstone: %s\n", tomb.Seq, tomb.Did) + } + + return nil + + }, // TODO: all the other event types Error: func(errf *events.ErrorFrame) error { return fmt.Errorf("error frame: %s: %s", errf.Error, errf.Message) diff --git a/events/dbpersist.go b/events/dbpersist.go index f94833c5..923ebc1f 100644 --- a/events/dbpersist.go +++ b/events/dbpersist.go @@ -165,6 +165,8 @@ func (p *DbPersistence) flushBatchLocked(ctx context.Context) error { e.RepoCommit.Seq = int64(item.Seq) case e.RepoHandle != nil: e.RepoHandle.Seq = int64(item.Seq) + case e.RepoTombstone != nil: + e.RepoTombstone.Seq = int64(item.Seq) default: return fmt.Errorf("unknown event type") } @@ -209,6 +211,11 @@ func (p *DbPersistence) Persist(ctx context.Context, e *XRPCStreamEvent) error { if err != nil { return err } + case e.RepoTombstone != nil: + rer, err = p.RecordFromTombstone(ctx, e.RepoTombstone) + if err != nil { + return err + } default: return nil } @@ -239,6 +246,24 @@ func (p *DbPersistence) RecordFromHandleChange(ctx context.Context, evt *comatpr }, nil } +func (p *DbPersistence) RecordFromTombstone(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Tombstone) (*RepoEventRecord, error) { + t, err := time.Parse(util.ISO8601, evt.Time) + if err != nil { + return nil, err + } + + uid, err := p.uidForDid(ctx, evt.Did) + if err != nil { + return nil, err + } + + return &RepoEventRecord{ + Repo: uid, + Type: "repo_tombstone", + Time: t, + }, nil +} + func (p *DbPersistence) RecordFromRepoCommit(ctx context.Context, evt *comatproto.SyncSubscribeRepos_Commit) (*RepoEventRecord, error) { // TODO: hack hack hack if len(evt.Ops) > 8192 { @@ -371,6 +396,8 @@ func (p *DbPersistence) hydrateBatch(ctx context.Context, batch []*RepoEventReco streamEvent, err = p.hydrateCommit(ctx, record) case record.NewHandle != nil: streamEvent, err = p.hydrateHandleChange(ctx, record) + case record.Type == "repo_tombstone": + streamEvent, err = p.hydrateTombstone(ctx, record) default: err = fmt.Errorf("unknown event type: %s", record.Type) } @@ -452,6 +479,20 @@ func (p *DbPersistence) hydrateHandleChange(ctx context.Context, rer *RepoEventR }, nil } +func (p *DbPersistence) hydrateTombstone(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) { + did, err := p.didForUid(ctx, rer.Repo) + if err != nil { + return nil, err + } + + return &XRPCStreamEvent{ + RepoTombstone: &comatproto.SyncSubscribeRepos_Tombstone{ + Did: did, + Time: rer.Time.Format(util.ISO8601), + }, + }, nil +} + func (p *DbPersistence) hydrateCommit(ctx context.Context, rer *RepoEventRecord) (*XRPCStreamEvent, error) { if rer.Commit == nil { return nil, fmt.Errorf("commit is nil") diff --git a/events/diskpersist.go b/events/diskpersist.go index dcf10361..a23cfb1d 100644 --- a/events/diskpersist.go +++ b/events/diskpersist.go @@ -273,8 +273,9 @@ func scanForLastSeq(fi *os.File, end int64) (int64, error) { } const ( - evtKindCommit = 1 - evtKindHandle = 2 + evtKindCommit = 1 + evtKindHandle = 2 + evtKindTombstone = 3 ) var emptyHeader = make([]byte, headerSize) @@ -450,8 +451,10 @@ func (dp *DiskPersistence) doPersist(ctx context.Context, j persistJob) error { e.RepoCommit.Seq = seq case e.RepoHandle != nil: e.RepoHandle.Seq = seq + case e.RepoTombstone != nil: + e.RepoTombstone.Seq = seq default: - // only those two get peristed right now + // only those three get peristed right now // we shouldnt actually ever get here... return nil } @@ -502,6 +505,12 @@ func (dp *DiskPersistence) Persist(ctx context.Context, e *XRPCStreamEvent) erro if err := e.RepoHandle.MarshalCBOR(cw); err != nil { return fmt.Errorf("failed to marshal: %w", err) } + case e.RepoTombstone != nil: + evtKind = evtKindTombstone + did = e.RepoTombstone.Did + if err := e.RepoTombstone.MarshalCBOR(cw); err != nil { + return fmt.Errorf("failed to marshal: %w", err) + } default: return nil // only those two get peristed right now @@ -691,6 +700,15 @@ func (dp *DiskPersistence) readEventsFrom(ctx context.Context, since int64, fn s if err := cb(&XRPCStreamEvent{RepoHandle: &evt}); err != nil { return err } + case evtKindTombstone: + var evt atproto.SyncSubscribeRepos_Tombstone + if err := evt.UnmarshalCBOR(io.LimitReader(bufr, h.Len64())); err != nil { + return err + } + evt.Seq = h.Seq + if err := cb(&XRPCStreamEvent{RepoTombstone: &evt}); err != nil { + return err + } default: log.Warnw("unrecognized event kind coming from log file", "seq", h.Seq, "kind", h.Kind) return fmt.Errorf("halting on unrecognized event kind") -- 2.51.2 From 876a9f4fa8932b52f628cea3f5cb95c320611153 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Wed, 4 Oct 2023 16:02:58 -0700 Subject: [PATCH 104/192] add commands for deleting accounts --- cmd/gosky/admin.go | 3 ++- cmd/gosky/main.go | 44 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/cmd/gosky/admin.go b/cmd/gosky/admin.go index 8134897d..a954b20c 100644 --- a/cmd/gosky/admin.go +++ b/cmd/gosky/admin.go @@ -103,7 +103,8 @@ var checkUserCmd = &cli.Command{ } else { handle, _, err := api.ResolveDidToHandle(ctx, xrpcc, plcc, phr, fa) if err != nil { - return fmt.Errorf("resolve did %q: %w", fa, err) + fmt.Println("ERROR: failed to resolve inviter: ", err) + handle = fa } invby = handle diff --git a/cmd/gosky/main.go b/cmd/gosky/main.go index aaec5093..8b240b17 100644 --- a/cmd/gosky/main.go +++ b/cmd/gosky/main.go @@ -94,7 +94,6 @@ func run(args []string) { refreshAuthTokenCmd, syncCmd, listAllPostsCmd, - deletePostCmd, getNotificationsCmd, followsCmd, resetPasswordCmd, @@ -105,6 +104,8 @@ func run(args []string) { adminCmd, createFeedGeneratorCmd, rebaseRepoCmd, + requestAccountDeletionCmd, + deleteAccountCmd, } app.RunAndExitOnError() @@ -1566,3 +1567,44 @@ var rebaseRepoCmd = &cli.Command{ return nil }, } + +var requestAccountDeletionCmd = &cli.Command{ + Name: "request-account-deletion", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + err = comatproto.ServerRequestAccountDelete(context.TODO(), xrpcc) + if err != nil { + return err + } + + return nil + }, +} + +var deleteAccountCmd = &cli.Command{ + Name: "delete-account", + Action: func(cctx *cli.Context) error { + xrpcc, err := cliutil.GetXrpcClient(cctx, false) + if err != nil { + return err + } + + token := cctx.Args().First() + password := cctx.Args().Get(1) + + err = comatproto.ServerDeleteAccount(context.TODO(), xrpcc, &comatproto.ServerDeleteAccount_Input{ + Did: xrpcc.Auth.Did, + Token: token, + Password: password, + }) + if err != nil { + return err + } + + return nil + }, +} -- 2.51.2 From f0a8a20119759f3c7d41716400793e28013770cf Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 04:58:01 +0000 Subject: [PATCH 105/192] Resync with a PDS by comparing revs insted of commit CIDs --- api/atproto/synclistRepos.go | 1 + bgs/bgs.go | 44 ++++++++++++++++++------------------ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/api/atproto/synclistRepos.go b/api/atproto/synclistRepos.go index 91ffd123..373aacb4 100644 --- a/api/atproto/synclistRepos.go +++ b/api/atproto/synclistRepos.go @@ -20,6 +20,7 @@ type SyncListRepos_Output struct { type SyncListRepos_Repo struct { Did string `json:"did" cborgen:"did"` Head string `json:"head" cborgen:"head"` + Rev string `json:"rev" cborgen:"rev"` } // SyncListRepos calls the XRPC method "com.atproto.sync.listRepos". diff --git a/bgs/bgs.go b/bgs/bgs.go index 04d475ae..94579bc7 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1250,12 +1250,12 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp }, nil } -type repoHead struct { - Did string - Head string +type repoRev struct { + Did string + Rev string } -type headCheckResult struct { +type revCheckResult struct { ai *models.ActorInfo err error } @@ -1343,7 +1343,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { cursor := "" limit := int64(500) - repos := []repoHead{} + repos := []repoRev{} pages := 0 @@ -1367,9 +1367,9 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } for _, r := range repoList.Repos { - repos = append(repos, repoHead{ - Did: r.Did, - Head: r.Head, + repos = append(repos, repoRev{ + Did: r.Did, + Rev: r.Rev, }) } @@ -1386,45 +1386,45 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { repolistDone := time.Now() log.Warnw("listed all repos, checking roots", "num_repos", len(repos), "took", repolistDone.Sub(start)) - resync = bgs.SetResyncStatus(pds.ID, "checking heads") + resync = bgs.SetResyncStatus(pds.ID, "checking revs") // Create a buffered channel for collecting results - results := make(chan headCheckResult, len(repos)) + results := make(chan revCheckResult, len(repos)) sem := semaphore.NewWeighted(40) - // Check repo heads against our local copy and enqueue crawls for any that are out of date + // Check repo revs against our local copy and enqueue crawls for any that are out of date for _, r := range repos { - go func(r repoHead) { + go func(r repoRev) { if err := sem.Acquire(ctx, 1); err != nil { log.Errorw("failed to acquire semaphore", "error", err) - results <- headCheckResult{err: err} + results <- revCheckResult{err: err} return } defer sem.Release(1) - log := log.With("did", r.Did, "head", r.Head) + log := log.With("did", r.Did, "remote_rev", r.Rev) // Fetches the user if we have it, otherwise automatically enqueues it for crawling ai, err := bgs.Index.GetUserOrMissing(ctx, r.Did) if err != nil { log.Errorw("failed to get user while resyncing PDS, we can't recrawl it", "error", err) - results <- headCheckResult{err: err} + results <- revCheckResult{err: err} return } - head, err := bgs.repoman.GetRepoRoot(ctx, ai.Uid) + rev, err := bgs.repoman.GetRepoRev(ctx, ai.Uid) if err != nil { log.Warnw("recrawling because we failed to get the local repo root", "err", err, "uid", ai.Uid) - results <- headCheckResult{ai: ai} + results <- revCheckResult{ai: ai} return } - if head.String() != r.Head { - log.Warnw("recrawling because the repo head from the PDS is different from our local repo root", "local_head", head.String()) - results <- headCheckResult{ai: ai} + if rev == "" || rev < r.Rev { + log.Warnw("recrawling because the repo rev from the PDS is newer than our local repo rev", "local_rev", rev) + results <- revCheckResult{ai: ai} return } - results <- headCheckResult{} + results <- revCheckResult{} }(r) } @@ -1443,7 +1443,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } } if i%10_000 == 0 { - log.Warnw("checked heads during resync", "num_repos_checked", i, "num_repos_to_crawl", numReposToResync, "took", time.Now().Sub(resync.StatusChangedAt)) + log.Warnw("checked revs during resync", "num_repos_checked", i, "num_repos_to_crawl", numReposToResync, "took", time.Now().Sub(resync.StatusChangedAt)) resync.NumReposChecked = i resync.NumReposToResync = numReposToResync bgs.UpdateResync(resync) -- 2.51.2 From 8e8cf4c3e287e56679c931a1a6c0880de5036b65 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 16:24:05 +0000 Subject: [PATCH 106/192] Move metrics to a different port for Palomar and BGS --- bgs/bgs.go | 16 ++++++++-------- cmd/bigsky/main.go | 14 +++++++++++++- cmd/palomar/main.go | 13 +++++++++++++ search/server.go | 6 ++++++ 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 04d475ae..4615bd3e 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -38,6 +38,7 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" promclient "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" dto "github.com/prometheus/client_model/go" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -142,6 +143,11 @@ func NewBGS(db *gorm.DB, ix *indexer.Indexer, repoman *repomgr.RepoManager, evtm return bgs, nil } +func (bgs *BGS) StartMetrics(listen string) error { + http.Handle("/metrics", promhttp.Handler()) + return http.ListenAndServe(listen, nil) +} + func (bgs *BGS) StartDebug(listen string) error { http.HandleFunc("/repodbg/user", func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() @@ -259,14 +265,14 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { e.Use(middleware.LoggerWithConfig(middleware.DefaultLoggerConfig)) } - e.Use(MetricsMiddleware) - // React uses a virtual router, so we need to serve the index.html for all // routes that aren't otherwise handled or in the /assets directory. e.File("/dash", "/public/index.html") e.File("/dash/*", "/public/index.html") e.Static("/assets", "/public/assets") + e.Use(MetricsMiddleware) + e.HTTPErrorHandler = func(err error, ctx echo.Context) { switch err := err.(type) { case *echo.HTTPError: @@ -309,12 +315,6 @@ func (bgs *BGS) StartWithListener(listen net.Listener) error { e.GET("/xrpc/com.atproto.sync.notifyOfUpdate", bgs.HandleComAtprotoSyncNotifyOfUpdate) e.GET("/xrpc/_health", bgs.HandleHealthCheck) - promh := prometheusHandler() - e.GET("/metrics", func(e echo.Context) error { - promh.ServeHTTP(e.Response().Writer, e.Request()) - return nil - }) - admin := e.Group("/admin", bgs.checkAdminAuth) // Slurper-related Admin API diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 2fa5f595..37a8342b 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -108,6 +108,11 @@ func run(args []string) { Name: "debug-listen", Value: "localhost:2471", }, + &cli.StringFlag{ + Name: "metrics-listen", + Value: ":2472", + EnvVars: []string{"BGS_METRICS_LISTEN"}, + }, &cli.StringFlag{ Name: "disk-blob-store", }, @@ -343,7 +348,14 @@ func Bigsky(cctx *cli.Context) error { // set up pprof endpoint go func() { if err := bgs.StartDebug(cctx.String("debug-listen")); err != nil { - panic(err) + log.Fatalf("failed to start debug endpoint: %s", err) + } + }() + + // set up metrics endpoint + go func() { + if err := bgs.StartMetrics(cctx.String("metrics-listen")); err != nil { + log.Fatalf("failed to start metrics endpoint: %s", err) } }() diff --git a/cmd/palomar/main.go b/cmd/palomar/main.go index 8b1a362b..f7a6887e 100644 --- a/cmd/palomar/main.go +++ b/cmd/palomar/main.go @@ -121,6 +121,12 @@ var runCmd = &cli.Command{ Value: ":3999", EnvVars: []string{"PALOMAR_BIND"}, }, + &cli.StringFlag{ + Name: "metrics-listen", + Usage: "IP or address, and port, to listen on for metrics APIs", + Value: ":3998", + EnvVars: []string{"PALOMAR_METRICS_LISTEN"}, + }, &cli.IntFlag{ Name: "bgs-sync-rate-limit", Usage: "max repo sync (checkout) requests per second to upstream (BGS)", @@ -185,6 +191,13 @@ var runCmd = &cli.Command{ return err } + go func() { + if err := srv.RunMetrics(cctx.String("metrics-listen")); err != nil { + slog.Error("failed to start metrics endpoint", "error", err) + panic(fmt.Errorf("failed to start metrics endpoint: %w", err)) + } + }() + go func() { srv.RunAPI(cctx.String("bind")) }() diff --git a/search/server.go b/search/server.go index 82dbccd2..2106104f 100644 --- a/search/server.go +++ b/search/server.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log/slog" + "net/http" "os" "strings" @@ -207,6 +208,11 @@ func (s *Server) RunAPI(listen string) error { return s.echo.Start(listen) } +func (s *Server) RunMetrics(listen string) error { + http.Handle("/metrics", promhttp.Handler()) + return http.ListenAndServe(listen, nil) +} + func (s *Server) Shutdown(ctx context.Context) error { return s.echo.Shutdown(ctx) } -- 2.51.2 From 62999f579e3c7b0875643c3f3958485469947d6a Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 16:28:48 +0000 Subject: [PATCH 107/192] Use existing type for repo revs --- bgs/bgs.go | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 94579bc7..19828abd 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1250,11 +1250,6 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp }, nil } -type repoRev struct { - Did string - Rev string -} - type revCheckResult struct { ai *models.ActorInfo err error @@ -1343,7 +1338,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { cursor := "" limit := int64(500) - repos := []repoRev{} + repos := []comatproto.SyncListRepos_Repo{} pages := 0 @@ -1367,7 +1362,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } for _, r := range repoList.Repos { - repos = append(repos, repoRev{ + repos = append(repos, comatproto.SyncListRepos_Repo{ Did: r.Did, Rev: r.Rev, }) @@ -1394,7 +1389,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { // Check repo revs against our local copy and enqueue crawls for any that are out of date for _, r := range repos { - go func(r repoRev) { + go func(r comatproto.SyncListRepos_Repo) { if err := sem.Acquire(ctx, 1); err != nil { log.Errorw("failed to acquire semaphore", "error", err) results <- revCheckResult{err: err} -- 2.51.2 From 5f44bcfa05b9d25fb1694e5c25bb2679585d288e Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 16:31:50 +0000 Subject: [PATCH 108/192] Clean up a lil --- bgs/bgs.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 19828abd..f2e647b9 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1362,10 +1362,9 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { } for _, r := range repoList.Repos { - repos = append(repos, comatproto.SyncListRepos_Repo{ - Did: r.Did, - Rev: r.Rev, - }) + if r != nil { + repos = append(repos, *r) + } } if repoList.Cursor == nil || *repoList.Cursor == "" { -- 2.51.2 From e4d0b551b7cfdaf58ec7992451966c7c0984194d Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 16:35:05 +0000 Subject: [PATCH 109/192] Reduce logging frequency --- bgs/bgs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index f2e647b9..f0e88d7f 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1436,7 +1436,7 @@ func (bgs *BGS) ResyncPDS(ctx context.Context, pds models.PDS) error { log.Errorw("failed to enqueue crawl for repo during resync", "error", err, "uid", res.ai.Uid, "did", res.ai.Did) } } - if i%10_000 == 0 { + if i%100_000 == 0 { log.Warnw("checked revs during resync", "num_repos_checked", i, "num_repos_to_crawl", numReposToResync, "took", time.Now().Sub(resync.StatusChangedAt)) resync.NumReposChecked = i resync.NumReposToResync = numReposToResync -- 2.51.2 From 19ca7959fbf75b388323601aed111d618c639262 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 16:42:53 +0000 Subject: [PATCH 110/192] Remove debug listener for BGS, pprof is accessible on the metrics server --- bgs/bgs.go | 1 + cmd/bigsky/main.go | 13 +------------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 4615bd3e..6dccb067 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -148,6 +148,7 @@ func (bgs *BGS) StartMetrics(listen string) error { return http.ListenAndServe(listen, nil) } +// Disabled for now, maybe reimplement behind admin auth later func (bgs *BGS) StartDebug(listen string) error { http.HandleFunc("/repodbg/user", func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 37a8342b..b11aa6d8 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -104,13 +104,9 @@ func run(args []string) { Name: "api-listen", Value: ":2470", }, - &cli.StringFlag{ - Name: "debug-listen", - Value: "localhost:2471", - }, &cli.StringFlag{ Name: "metrics-listen", - Value: ":2472", + Value: ":2471", EnvVars: []string{"BGS_METRICS_LISTEN"}, }, &cli.StringFlag{ @@ -345,13 +341,6 @@ func Bigsky(cctx *cli.Context) error { } } - // set up pprof endpoint - go func() { - if err := bgs.StartDebug(cctx.String("debug-listen")); err != nil { - log.Fatalf("failed to start debug endpoint: %s", err) - } - }() - // set up metrics endpoint go func() { if err := bgs.StartMetrics(cctx.String("metrics-listen")); err != nil { -- 2.51.2 From 0fba1fb3d3f84ed9575febf6bdf2d27bf324bf3a Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 17:48:13 +0000 Subject: [PATCH 111/192] Bump lexgen and make empty lexbytes omit in cbor --- api/atproto/serverconfirmEmail.go | 26 +++++++ api/atproto/servercreateSession.go | 11 +-- api/atproto/servergetSession.go | 7 +- api/atproto/serverrequestEmailConfirmation.go | 20 ++++++ api/atproto/serverrequestEmailUpdate.go | 26 +++++++ api/atproto/serverupdateEmail.go | 27 ++++++++ api/atproto/syncsubscribeRepos.go | 2 +- api/bsky/actordefs.go | 69 ++++++++++++++++++- api/bsky/embedimages.go | 20 ++++-- api/bsky/feedgetListFeed.go | 33 +++++++++ api/bsky/feedgetSuggestedFeeds.go | 32 +++++++++ api/bsky/graphdefs.go | 3 +- api/bsky/graphgetSuggestedFollowsByActor.go | 30 ++++++++ lex/gen.go | 1 + 14 files changed, 289 insertions(+), 18 deletions(-) create mode 100644 api/atproto/serverconfirmEmail.go create mode 100644 api/atproto/serverrequestEmailConfirmation.go create mode 100644 api/atproto/serverrequestEmailUpdate.go create mode 100644 api/atproto/serverupdateEmail.go create mode 100644 api/bsky/feedgetListFeed.go create mode 100644 api/bsky/feedgetSuggestedFeeds.go create mode 100644 api/bsky/graphgetSuggestedFollowsByActor.go diff --git a/api/atproto/serverconfirmEmail.go b/api/atproto/serverconfirmEmail.go new file mode 100644 index 00000000..c3c730a0 --- /dev/null +++ b/api/atproto/serverconfirmEmail.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package atproto + +// schema: com.atproto.server.confirmEmail + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// ServerConfirmEmail_Input is the input argument to a com.atproto.server.confirmEmail call. +type ServerConfirmEmail_Input struct { + Email string `json:"email" cborgen:"email"` + Token string `json:"token" cborgen:"token"` +} + +// ServerConfirmEmail calls the XRPC method "com.atproto.server.confirmEmail". +func ServerConfirmEmail(ctx context.Context, c *xrpc.Client, input *ServerConfirmEmail_Input) error { + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.server.confirmEmail", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/servercreateSession.go b/api/atproto/servercreateSession.go index 7f44022a..269108a0 100644 --- a/api/atproto/servercreateSession.go +++ b/api/atproto/servercreateSession.go @@ -19,11 +19,12 @@ type ServerCreateSession_Input struct { // ServerCreateSession_Output is the output of a com.atproto.server.createSession call. type ServerCreateSession_Output struct { - AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` - Did string `json:"did" cborgen:"did"` - Email *string `json:"email,omitempty" cborgen:"email,omitempty"` - Handle string `json:"handle" cborgen:"handle"` - RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` + AccessJwt string `json:"accessJwt" cborgen:"accessJwt"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` + Handle string `json:"handle" cborgen:"handle"` + RefreshJwt string `json:"refreshJwt" cborgen:"refreshJwt"` } // ServerCreateSession calls the XRPC method "com.atproto.server.createSession". diff --git a/api/atproto/servergetSession.go b/api/atproto/servergetSession.go index 12ebc538..c9cd5b28 100644 --- a/api/atproto/servergetSession.go +++ b/api/atproto/servergetSession.go @@ -12,9 +12,10 @@ import ( // ServerGetSession_Output is the output of a com.atproto.server.getSession call. type ServerGetSession_Output struct { - Did string `json:"did" cborgen:"did"` - Email *string `json:"email,omitempty" cborgen:"email,omitempty"` - Handle string `json:"handle" cborgen:"handle"` + Did string `json:"did" cborgen:"did"` + Email *string `json:"email,omitempty" cborgen:"email,omitempty"` + EmailConfirmed *bool `json:"emailConfirmed,omitempty" cborgen:"emailConfirmed,omitempty"` + Handle string `json:"handle" cborgen:"handle"` } // ServerGetSession calls the XRPC method "com.atproto.server.getSession". diff --git a/api/atproto/serverrequestEmailConfirmation.go b/api/atproto/serverrequestEmailConfirmation.go new file mode 100644 index 00000000..a9945874 --- /dev/null +++ b/api/atproto/serverrequestEmailConfirmation.go @@ -0,0 +1,20 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package atproto + +// schema: com.atproto.server.requestEmailConfirmation + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// ServerRequestEmailConfirmation calls the XRPC method "com.atproto.server.requestEmailConfirmation". +func ServerRequestEmailConfirmation(ctx context.Context, c *xrpc.Client) error { + if err := c.Do(ctx, xrpc.Procedure, "", "com.atproto.server.requestEmailConfirmation", nil, nil, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/serverrequestEmailUpdate.go b/api/atproto/serverrequestEmailUpdate.go new file mode 100644 index 00000000..23045883 --- /dev/null +++ b/api/atproto/serverrequestEmailUpdate.go @@ -0,0 +1,26 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package atproto + +// schema: com.atproto.server.requestEmailUpdate + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// ServerRequestEmailUpdate_Output is the output of a com.atproto.server.requestEmailUpdate call. +type ServerRequestEmailUpdate_Output struct { + TokenRequired bool `json:"tokenRequired" cborgen:"tokenRequired"` +} + +// ServerRequestEmailUpdate calls the XRPC method "com.atproto.server.requestEmailUpdate". +func ServerRequestEmailUpdate(ctx context.Context, c *xrpc.Client) (*ServerRequestEmailUpdate_Output, error) { + var out ServerRequestEmailUpdate_Output + if err := c.Do(ctx, xrpc.Procedure, "", "com.atproto.server.requestEmailUpdate", nil, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/atproto/serverupdateEmail.go b/api/atproto/serverupdateEmail.go new file mode 100644 index 00000000..3b22a00c --- /dev/null +++ b/api/atproto/serverupdateEmail.go @@ -0,0 +1,27 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package atproto + +// schema: com.atproto.server.updateEmail + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// ServerUpdateEmail_Input is the input argument to a com.atproto.server.updateEmail call. +type ServerUpdateEmail_Input struct { + Email string `json:"email" cborgen:"email"` + // token: Requires a token from com.atproto.sever.requestEmailUpdate if the account's email has been confirmed. + Token *string `json:"token,omitempty" cborgen:"token,omitempty"` +} + +// ServerUpdateEmail calls the XRPC method "com.atproto.server.updateEmail". +func ServerUpdateEmail(ctx context.Context, c *xrpc.Client, input *ServerUpdateEmail_Input) error { + if err := c.Do(ctx, xrpc.Procedure, "application/json", "com.atproto.server.updateEmail", nil, input, nil); err != nil { + return err + } + + return nil +} diff --git a/api/atproto/syncsubscribeRepos.go b/api/atproto/syncsubscribeRepos.go index 47a014bb..43404873 100644 --- a/api/atproto/syncsubscribeRepos.go +++ b/api/atproto/syncsubscribeRepos.go @@ -12,7 +12,7 @@ import ( type SyncSubscribeRepos_Commit struct { Blobs []util.LexLink `json:"blobs" cborgen:"blobs"` // blocks: CAR file containing relevant blocks - Blocks util.LexBytes `json:"blocks,omitempty" cborgen:"blocks"` + Blocks util.LexBytes `json:"blocks,omitempty" cborgen:"blocks,omitempty"` Commit util.LexLink `json:"commit" cborgen:"commit"` Ops []*SyncSubscribeRepos_RepoOp `json:"ops" cborgen:"ops"` Prev *util.LexLink `json:"prev" cborgen:"prev"` diff --git a/api/bsky/actordefs.go b/api/bsky/actordefs.go index 0e9b88f3..357048c0 100644 --- a/api/bsky/actordefs.go +++ b/api/bsky/actordefs.go @@ -29,10 +29,41 @@ type ActorDefs_ContentLabelPref struct { Visibility string `json:"visibility" cborgen:"visibility"` } +// ActorDefs_FeedViewPref is a "feedViewPref" in the app.bsky.actor.defs schema. +// +// RECORDTYPE: ActorDefs_FeedViewPref +type ActorDefs_FeedViewPref struct { + LexiconTypeID string `json:"$type,const=app.bsky.actor.defs#feedViewPref" cborgen:"$type,const=app.bsky.actor.defs#feedViewPref"` + // feed: The URI of the feed, or an identifier which describes the feed. + Feed string `json:"feed" cborgen:"feed"` + // hideQuotePosts: Hide quote posts in the feed. + HideQuotePosts *bool `json:"hideQuotePosts,omitempty" cborgen:"hideQuotePosts,omitempty"` + // hideReplies: Hide replies in the feed. + HideReplies *bool `json:"hideReplies,omitempty" cborgen:"hideReplies,omitempty"` + // hideRepliesByLikeCount: Hide replies in the feed if they do not have this number of likes. + HideRepliesByLikeCount *int64 `json:"hideRepliesByLikeCount,omitempty" cborgen:"hideRepliesByLikeCount,omitempty"` + // hideRepliesByUnfollowed: Hide replies in the feed if they are not by followed users. + HideRepliesByUnfollowed *bool `json:"hideRepliesByUnfollowed,omitempty" cborgen:"hideRepliesByUnfollowed,omitempty"` + // hideReposts: Hide reposts in the feed. + HideReposts *bool `json:"hideReposts,omitempty" cborgen:"hideReposts,omitempty"` +} + +// ActorDefs_PersonalDetailsPref is a "personalDetailsPref" in the app.bsky.actor.defs schema. +// +// RECORDTYPE: ActorDefs_PersonalDetailsPref +type ActorDefs_PersonalDetailsPref struct { + LexiconTypeID string `json:"$type,const=app.bsky.actor.defs#personalDetailsPref" cborgen:"$type,const=app.bsky.actor.defs#personalDetailsPref"` + // birthDate: The birth date of the owner of the account. + BirthDate *string `json:"birthDate,omitempty" cborgen:"birthDate,omitempty"` +} + type ActorDefs_Preferences_Elem struct { - ActorDefs_AdultContentPref *ActorDefs_AdultContentPref - ActorDefs_ContentLabelPref *ActorDefs_ContentLabelPref - ActorDefs_SavedFeedsPref *ActorDefs_SavedFeedsPref + ActorDefs_AdultContentPref *ActorDefs_AdultContentPref + ActorDefs_ContentLabelPref *ActorDefs_ContentLabelPref + ActorDefs_SavedFeedsPref *ActorDefs_SavedFeedsPref + ActorDefs_PersonalDetailsPref *ActorDefs_PersonalDetailsPref + ActorDefs_FeedViewPref *ActorDefs_FeedViewPref + ActorDefs_ThreadViewPref *ActorDefs_ThreadViewPref } func (t *ActorDefs_Preferences_Elem) MarshalJSON() ([]byte, error) { @@ -48,6 +79,18 @@ func (t *ActorDefs_Preferences_Elem) MarshalJSON() ([]byte, error) { t.ActorDefs_SavedFeedsPref.LexiconTypeID = "app.bsky.actor.defs#savedFeedsPref" return json.Marshal(t.ActorDefs_SavedFeedsPref) } + if t.ActorDefs_PersonalDetailsPref != nil { + t.ActorDefs_PersonalDetailsPref.LexiconTypeID = "app.bsky.actor.defs#personalDetailsPref" + return json.Marshal(t.ActorDefs_PersonalDetailsPref) + } + if t.ActorDefs_FeedViewPref != nil { + t.ActorDefs_FeedViewPref.LexiconTypeID = "app.bsky.actor.defs#feedViewPref" + return json.Marshal(t.ActorDefs_FeedViewPref) + } + if t.ActorDefs_ThreadViewPref != nil { + t.ActorDefs_ThreadViewPref.LexiconTypeID = "app.bsky.actor.defs#threadViewPref" + return json.Marshal(t.ActorDefs_ThreadViewPref) + } return nil, fmt.Errorf("cannot marshal empty enum") } func (t *ActorDefs_Preferences_Elem) UnmarshalJSON(b []byte) error { @@ -66,6 +109,15 @@ func (t *ActorDefs_Preferences_Elem) UnmarshalJSON(b []byte) error { case "app.bsky.actor.defs#savedFeedsPref": t.ActorDefs_SavedFeedsPref = new(ActorDefs_SavedFeedsPref) return json.Unmarshal(b, t.ActorDefs_SavedFeedsPref) + case "app.bsky.actor.defs#personalDetailsPref": + t.ActorDefs_PersonalDetailsPref = new(ActorDefs_PersonalDetailsPref) + return json.Unmarshal(b, t.ActorDefs_PersonalDetailsPref) + case "app.bsky.actor.defs#feedViewPref": + t.ActorDefs_FeedViewPref = new(ActorDefs_FeedViewPref) + return json.Unmarshal(b, t.ActorDefs_FeedViewPref) + case "app.bsky.actor.defs#threadViewPref": + t.ActorDefs_ThreadViewPref = new(ActorDefs_ThreadViewPref) + return json.Unmarshal(b, t.ActorDefs_ThreadViewPref) default: return nil @@ -119,6 +171,17 @@ type ActorDefs_SavedFeedsPref struct { Saved []string `json:"saved" cborgen:"saved"` } +// ActorDefs_ThreadViewPref is a "threadViewPref" in the app.bsky.actor.defs schema. +// +// RECORDTYPE: ActorDefs_ThreadViewPref +type ActorDefs_ThreadViewPref struct { + LexiconTypeID string `json:"$type,const=app.bsky.actor.defs#threadViewPref" cborgen:"$type,const=app.bsky.actor.defs#threadViewPref"` + // prioritizeFollowedUsers: Show followed users at the top of all replies. + PrioritizeFollowedUsers *bool `json:"prioritizeFollowedUsers,omitempty" cborgen:"prioritizeFollowedUsers,omitempty"` + // sort: Sorting mode. + Sort *string `json:"sort,omitempty" cborgen:"sort,omitempty"` +} + // ActorDefs_ViewerState is a "viewerState" in the app.bsky.actor.defs schema. type ActorDefs_ViewerState struct { BlockedBy *bool `json:"blockedBy,omitempty" cborgen:"blockedBy,omitempty"` diff --git a/api/bsky/embedimages.go b/api/bsky/embedimages.go index 927f3ff7..06a6ca6f 100644 --- a/api/bsky/embedimages.go +++ b/api/bsky/embedimages.go @@ -17,10 +17,19 @@ type EmbedImages struct { Images []*EmbedImages_Image `json:"images" cborgen:"images"` } +// EmbedImages_AspectRatio is a "aspectRatio" in the app.bsky.embed.images schema. +// +// width:height represents an aspect ratio. It may be approximate, and may not correspond to absolute dimensions in any given unit. +type EmbedImages_AspectRatio struct { + Height int64 `json:"height" cborgen:"height"` + Width int64 `json:"width" cborgen:"width"` +} + // EmbedImages_Image is a "image" in the app.bsky.embed.images schema. type EmbedImages_Image struct { - Alt string `json:"alt" cborgen:"alt"` - Image *util.LexBlob `json:"image" cborgen:"image"` + Alt string `json:"alt" cborgen:"alt"` + AspectRatio *EmbedImages_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` + Image *util.LexBlob `json:"image" cborgen:"image"` } // EmbedImages_View is a "view" in the app.bsky.embed.images schema. @@ -33,7 +42,8 @@ type EmbedImages_View struct { // EmbedImages_ViewImage is a "viewImage" in the app.bsky.embed.images schema. type EmbedImages_ViewImage struct { - Alt string `json:"alt" cborgen:"alt"` - Fullsize string `json:"fullsize" cborgen:"fullsize"` - Thumb string `json:"thumb" cborgen:"thumb"` + Alt string `json:"alt" cborgen:"alt"` + AspectRatio *EmbedImages_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` + Fullsize string `json:"fullsize" cborgen:"fullsize"` + Thumb string `json:"thumb" cborgen:"thumb"` } diff --git a/api/bsky/feedgetListFeed.go b/api/bsky/feedgetListFeed.go new file mode 100644 index 00000000..634616cc --- /dev/null +++ b/api/bsky/feedgetListFeed.go @@ -0,0 +1,33 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.feed.getListFeed + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// FeedGetListFeed_Output is the output of a app.bsky.feed.getListFeed call. +type FeedGetListFeed_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feed []*FeedDefs_FeedViewPost `json:"feed" cborgen:"feed"` +} + +// FeedGetListFeed calls the XRPC method "app.bsky.feed.getListFeed". +func FeedGetListFeed(ctx context.Context, c *xrpc.Client, cursor string, limit int64, list string) (*FeedGetListFeed_Output, error) { + var out FeedGetListFeed_Output + + params := map[string]interface{}{ + "cursor": cursor, + "limit": limit, + "list": list, + } + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getListFeed", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/feedgetSuggestedFeeds.go b/api/bsky/feedgetSuggestedFeeds.go new file mode 100644 index 00000000..7c16eaed --- /dev/null +++ b/api/bsky/feedgetSuggestedFeeds.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.feed.getSuggestedFeeds + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// FeedGetSuggestedFeeds_Output is the output of a app.bsky.feed.getSuggestedFeeds call. +type FeedGetSuggestedFeeds_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Feeds []*FeedDefs_GeneratorView `json:"feeds" cborgen:"feeds"` +} + +// FeedGetSuggestedFeeds calls the XRPC method "app.bsky.feed.getSuggestedFeeds". +func FeedGetSuggestedFeeds(ctx context.Context, c *xrpc.Client, cursor string, limit int64) (*FeedGetSuggestedFeeds_Output, error) { + var out FeedGetSuggestedFeeds_Output + + params := map[string]interface{}{ + "cursor": cursor, + "limit": limit, + } + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.feed.getSuggestedFeeds", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphdefs.go b/api/bsky/graphdefs.go index 46db29bd..a662b2ba 100644 --- a/api/bsky/graphdefs.go +++ b/api/bsky/graphdefs.go @@ -39,5 +39,6 @@ type GraphDefs_ListViewBasic struct { // GraphDefs_ListViewerState is a "listViewerState" in the app.bsky.graph.defs schema. type GraphDefs_ListViewerState struct { - Muted *bool `json:"muted,omitempty" cborgen:"muted,omitempty"` + Blocked *string `json:"blocked,omitempty" cborgen:"blocked,omitempty"` + Muted *bool `json:"muted,omitempty" cborgen:"muted,omitempty"` } diff --git a/api/bsky/graphgetSuggestedFollowsByActor.go b/api/bsky/graphgetSuggestedFollowsByActor.go new file mode 100644 index 00000000..58a33411 --- /dev/null +++ b/api/bsky/graphgetSuggestedFollowsByActor.go @@ -0,0 +1,30 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.graph.getSuggestedFollowsByActor + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// GraphGetSuggestedFollowsByActor_Output is the output of a app.bsky.graph.getSuggestedFollowsByActor call. +type GraphGetSuggestedFollowsByActor_Output struct { + Suggestions []*ActorDefs_ProfileView `json:"suggestions" cborgen:"suggestions"` +} + +// GraphGetSuggestedFollowsByActor calls the XRPC method "app.bsky.graph.getSuggestedFollowsByActor". +func GraphGetSuggestedFollowsByActor(ctx context.Context, c *xrpc.Client, actor string) (*GraphGetSuggestedFollowsByActor_Output, error) { + var out GraphGetSuggestedFollowsByActor_Output + + params := map[string]interface{}{ + "actor": actor, + } + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getSuggestedFollowsByActor", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/lex/gen.go b/lex/gen.go index 1777a336..55208cc8 100644 --- a/lex/gen.go +++ b/lex/gen.go @@ -1258,6 +1258,7 @@ func (ts *TypeSchema) writeTypeDefinition(name string, w io.Writer) error { // be omitempty. if ptr == "" && tname == "util.LexBytes" { jsonOmit = ",omitempty" + cborOmit = ",omitempty" } if v.Description != "" { -- 2.51.2 From a2f217c5eb1cb47b85c657a28c33e118e94371b7 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 18:08:14 +0000 Subject: [PATCH 112/192] Bump cbor gen --- api/atproto/cbor_gen.go | 44 +++++++++++++++++++++++---------------- api/bsky/cbor_gen.go | 46 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 71 insertions(+), 19 deletions(-) diff --git a/api/atproto/cbor_gen.go b/api/atproto/cbor_gen.go index 455bea18..23b39094 100644 --- a/api/atproto/cbor_gen.go +++ b/api/atproto/cbor_gen.go @@ -193,8 +193,13 @@ func (t *SyncSubscribeRepos_Commit) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 12 - if _, err := cw.Write([]byte{172}); err != nil { + if t.Blocks == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -385,27 +390,30 @@ func (t *SyncSubscribeRepos_Commit) MarshalCBOR(w io.Writer) error { } // t.Blocks (util.LexBytes) (slice) - if len("blocks") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"blocks\" was too long") - } + if t.Blocks != nil { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blocks"))); err != nil { - return err - } - if _, err := cw.WriteString(string("blocks")); err != nil { - return err - } + if len("blocks") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"blocks\" was too long") + } - if len(t.Blocks) > cbg.ByteArrayMaxLen { - return xerrors.Errorf("Byte array in field t.Blocks was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("blocks"))); err != nil { + return err + } + if _, err := cw.WriteString(string("blocks")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Blocks))); err != nil { - return err - } + if len(t.Blocks) > cbg.ByteArrayMaxLen { + return xerrors.Errorf("Byte array in field t.Blocks was too long") + } - if _, err := cw.Write(t.Blocks[:]); err != nil { - return err + if err := cw.WriteMajorTypeHeader(cbg.MajByteString, uint64(len(t.Blocks))); err != nil { + return err + } + + if _, err := cw.Write(t.Blocks[:]); err != nil { + return err + } } // t.Commit (util.LexLink) (struct) diff --git a/api/bsky/cbor_gen.go b/api/bsky/cbor_gen.go index d59d7f41..c93667df 100644 --- a/api/bsky/cbor_gen.go +++ b/api/bsky/cbor_gen.go @@ -1677,8 +1677,13 @@ func (t *EmbedImages_Image) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) + fieldCount := 3 - if _, err := cw.Write([]byte{162}); err != nil { + if t.AspectRatio == nil { + fieldCount-- + } + + if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { return err } @@ -1720,6 +1725,25 @@ func (t *EmbedImages_Image) MarshalCBOR(w io.Writer) error { if err := t.Image.MarshalCBOR(cw); err != nil { return err } + + // t.AspectRatio (bsky.EmbedImages_AspectRatio) (struct) + if t.AspectRatio != nil { + + if len("aspectRatio") > cbg.MaxLength { + return xerrors.Errorf("Value in field \"aspectRatio\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("aspectRatio"))); err != nil { + return err + } + if _, err := cw.WriteString(string("aspectRatio")); err != nil { + return err + } + + if err := t.AspectRatio.MarshalCBOR(cw); err != nil { + return err + } + } return nil } @@ -1792,6 +1816,26 @@ func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { } } + // t.AspectRatio (bsky.EmbedImages_AspectRatio) (struct) + case "aspectRatio": + + { + + b, err := cr.ReadByte() + if err != nil { + return err + } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } + t.AspectRatio = new(EmbedImages_AspectRatio) + if err := t.AspectRatio.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.AspectRatio pointer: %w", err) + } + } + + } default: // Field doesn't exist on this type, so ignore it -- 2.51.2 From 3da55504af7d74748fabb16bfc3a18b43965f470 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 18:14:07 +0000 Subject: [PATCH 113/192] More lexgen --- api/bsky/feeddefs.go | 15 ++++ api/bsky/feedthreadgate.go | 128 +++++++++++++++++++++++++++++++++ api/bsky/graphgetListBlocks.go | 32 +++++++++ api/bsky/graphlistblock.go | 19 +++++ 4 files changed, 194 insertions(+) create mode 100644 api/bsky/feedthreadgate.go create mode 100644 api/bsky/graphgetListBlocks.go create mode 100644 api/bsky/graphlistblock.go diff --git a/api/bsky/feeddefs.go b/api/bsky/feeddefs.go index a77e26ef..4fc61cbd 100644 --- a/api/bsky/feeddefs.go +++ b/api/bsky/feeddefs.go @@ -108,6 +108,7 @@ type FeedDefs_PostView struct { Record *util.LexiconTypeDecoder `json:"record" cborgen:"record"` ReplyCount *int64 `json:"replyCount,omitempty" cborgen:"replyCount,omitempty"` RepostCount *int64 `json:"repostCount,omitempty" cborgen:"repostCount,omitempty"` + Threadgate *FeedDefs_ThreadgateView `json:"threadgate,omitempty" cborgen:"threadgate,omitempty"` Uri string `json:"uri" cborgen:"uri"` Viewer *FeedDefs_ViewerState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` } @@ -313,6 +314,7 @@ type FeedDefs_ThreadViewPost struct { Parent *FeedDefs_ThreadViewPost_Parent `json:"parent,omitempty" cborgen:"parent,omitempty"` Post *FeedDefs_PostView `json:"post" cborgen:"post"` Replies []*FeedDefs_ThreadViewPost_Replies_Elem `json:"replies,omitempty" cborgen:"replies,omitempty"` + Viewer *FeedDefs_ViewerThreadState `json:"viewer,omitempty" cborgen:"viewer,omitempty"` } type FeedDefs_ThreadViewPost_Parent struct { @@ -401,8 +403,21 @@ func (t *FeedDefs_ThreadViewPost_Replies_Elem) UnmarshalJSON(b []byte) error { } } +// FeedDefs_ThreadgateView is a "threadgateView" in the app.bsky.feed.defs schema. +type FeedDefs_ThreadgateView struct { + Cid *string `json:"cid,omitempty" cborgen:"cid,omitempty"` + Lists []*GraphDefs_ListViewBasic `json:"lists,omitempty" cborgen:"lists,omitempty"` + Record *util.LexiconTypeDecoder `json:"record,omitempty" cborgen:"record,omitempty"` + Uri *string `json:"uri,omitempty" cborgen:"uri,omitempty"` +} + // FeedDefs_ViewerState is a "viewerState" in the app.bsky.feed.defs schema. type FeedDefs_ViewerState struct { Like *string `json:"like,omitempty" cborgen:"like,omitempty"` Repost *string `json:"repost,omitempty" cborgen:"repost,omitempty"` } + +// FeedDefs_ViewerThreadState is a "viewerThreadState" in the app.bsky.feed.defs schema. +type FeedDefs_ViewerThreadState struct { + CanReply *bool `json:"canReply,omitempty" cborgen:"canReply,omitempty"` +} diff --git a/api/bsky/feedthreadgate.go b/api/bsky/feedthreadgate.go new file mode 100644 index 00000000..14d45393 --- /dev/null +++ b/api/bsky/feedthreadgate.go @@ -0,0 +1,128 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.feed.threadgate + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" +) + +func init() { + util.RegisterType("app.bsky.feed.threadgate", &FeedThreadgate{}) +} // +// RECORDTYPE: FeedThreadgate +type FeedThreadgate struct { + LexiconTypeID string `json:"$type,const=app.bsky.feed.threadgate" cborgen:"$type,const=app.bsky.feed.threadgate"` + Allow []*FeedThreadgate_Allow_Elem `json:"allow,omitempty" cborgen:"allow,omitempty"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Post string `json:"post" cborgen:"post"` +} + +type FeedThreadgate_Allow_Elem struct { + FeedThreadgate_MentionRule *FeedThreadgate_MentionRule + FeedThreadgate_FollowingRule *FeedThreadgate_FollowingRule + FeedThreadgate_ListRule *FeedThreadgate_ListRule +} + +func (t *FeedThreadgate_Allow_Elem) MarshalJSON() ([]byte, error) { + if t.FeedThreadgate_MentionRule != nil { + t.FeedThreadgate_MentionRule.LexiconTypeID = "app.bsky.feed.threadgate#mentionRule" + return json.Marshal(t.FeedThreadgate_MentionRule) + } + if t.FeedThreadgate_FollowingRule != nil { + t.FeedThreadgate_FollowingRule.LexiconTypeID = "app.bsky.feed.threadgate#followingRule" + return json.Marshal(t.FeedThreadgate_FollowingRule) + } + if t.FeedThreadgate_ListRule != nil { + t.FeedThreadgate_ListRule.LexiconTypeID = "app.bsky.feed.threadgate#listRule" + return json.Marshal(t.FeedThreadgate_ListRule) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *FeedThreadgate_Allow_Elem) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.threadgate#mentionRule": + t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) + return json.Unmarshal(b, t.FeedThreadgate_MentionRule) + case "app.bsky.feed.threadgate#followingRule": + t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) + return json.Unmarshal(b, t.FeedThreadgate_FollowingRule) + case "app.bsky.feed.threadgate#listRule": + t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) + return json.Unmarshal(b, t.FeedThreadgate_ListRule) + + default: + return nil + } +} + +func (t *FeedThreadgate_Allow_Elem) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.FeedThreadgate_MentionRule != nil { + return t.FeedThreadgate_MentionRule.MarshalCBOR(w) + } + if t.FeedThreadgate_FollowingRule != nil { + return t.FeedThreadgate_FollowingRule.MarshalCBOR(w) + } + if t.FeedThreadgate_ListRule != nil { + return t.FeedThreadgate_ListRule.MarshalCBOR(w) + } + return fmt.Errorf("cannot cbor marshal empty enum") +} +func (t *FeedThreadgate_Allow_Elem) UnmarshalCBOR(r io.Reader) error { + typ, b, err := util.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "app.bsky.feed.threadgate#mentionRule": + t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) + return t.FeedThreadgate_MentionRule.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.feed.threadgate#followingRule": + t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) + return t.FeedThreadgate_FollowingRule.UnmarshalCBOR(bytes.NewReader(b)) + case "app.bsky.feed.threadgate#listRule": + t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) + return t.FeedThreadgate_ListRule.UnmarshalCBOR(bytes.NewReader(b)) + + default: + return nil + } +} + +// FeedThreadgate_FollowingRule is a "followingRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors you follow. +type FeedThreadgate_FollowingRule interface{} + +// FeedThreadgate_ListRule is a "listRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors on a list. +// +// RECORDTYPE: FeedThreadgate_ListRule +type FeedThreadgate_ListRule struct { + LexiconTypeID string `json:"$type,const=app.bsky.feed.threadgate#listRule" cborgen:"$type,const=app.bsky.feed.threadgate#listRule"` + List string `json:"list" cborgen:"list"` +} + +// FeedThreadgate_MentionRule is a "mentionRule" in the app.bsky.feed.threadgate schema. +// +// Allow replies from actors mentioned in your post. +type FeedThreadgate_MentionRule interface{} diff --git a/api/bsky/graphgetListBlocks.go b/api/bsky/graphgetListBlocks.go new file mode 100644 index 00000000..649c8254 --- /dev/null +++ b/api/bsky/graphgetListBlocks.go @@ -0,0 +1,32 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.graph.getListBlocks + +import ( + "context" + + "github.com/bluesky-social/indigo/xrpc" +) + +// GraphGetListBlocks_Output is the output of a app.bsky.graph.getListBlocks call. +type GraphGetListBlocks_Output struct { + Cursor *string `json:"cursor,omitempty" cborgen:"cursor,omitempty"` + Lists []*GraphDefs_ListView `json:"lists" cborgen:"lists"` +} + +// GraphGetListBlocks calls the XRPC method "app.bsky.graph.getListBlocks". +func GraphGetListBlocks(ctx context.Context, c *xrpc.Client, cursor string, limit int64) (*GraphGetListBlocks_Output, error) { + var out GraphGetListBlocks_Output + + params := map[string]interface{}{ + "cursor": cursor, + "limit": limit, + } + if err := c.Do(ctx, xrpc.Query, "", "app.bsky.graph.getListBlocks", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/bsky/graphlistblock.go b/api/bsky/graphlistblock.go new file mode 100644 index 00000000..ee743007 --- /dev/null +++ b/api/bsky/graphlistblock.go @@ -0,0 +1,19 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package bsky + +// schema: app.bsky.graph.listblock + +import ( + "github.com/bluesky-social/indigo/lex/util" +) + +func init() { + util.RegisterType("app.bsky.graph.listblock", &GraphListblock{}) +} // +// RECORDTYPE: GraphListblock +type GraphListblock struct { + LexiconTypeID string `json:"$type,const=app.bsky.graph.listblock" cborgen:"$type,const=app.bsky.graph.listblock"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Subject string `json:"subject" cborgen:"subject"` +} -- 2.51.2 From c022e4ce14636dc5c10a7c5d298836d09a85aec2 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 18:37:53 +0000 Subject: [PATCH 114/192] Remove weird stuff from lexicons that don't make sense right now --- api/bsky/cbor_gen.go | 46 +------------ api/bsky/embedimages.go | 10 --- api/bsky/feedthreadgate.go | 128 ------------------------------------- api/bsky/graphlistblock.go | 19 ------ 4 files changed, 1 insertion(+), 202 deletions(-) delete mode 100644 api/bsky/feedthreadgate.go delete mode 100644 api/bsky/graphlistblock.go diff --git a/api/bsky/cbor_gen.go b/api/bsky/cbor_gen.go index c93667df..d59d7f41 100644 --- a/api/bsky/cbor_gen.go +++ b/api/bsky/cbor_gen.go @@ -1677,13 +1677,8 @@ func (t *EmbedImages_Image) MarshalCBOR(w io.Writer) error { } cw := cbg.NewCborWriter(w) - fieldCount := 3 - if t.AspectRatio == nil { - fieldCount-- - } - - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + if _, err := cw.Write([]byte{162}); err != nil { return err } @@ -1725,25 +1720,6 @@ func (t *EmbedImages_Image) MarshalCBOR(w io.Writer) error { if err := t.Image.MarshalCBOR(cw); err != nil { return err } - - // t.AspectRatio (bsky.EmbedImages_AspectRatio) (struct) - if t.AspectRatio != nil { - - if len("aspectRatio") > cbg.MaxLength { - return xerrors.Errorf("Value in field \"aspectRatio\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("aspectRatio"))); err != nil { - return err - } - if _, err := cw.WriteString(string("aspectRatio")); err != nil { - return err - } - - if err := t.AspectRatio.MarshalCBOR(cw); err != nil { - return err - } - } return nil } @@ -1816,26 +1792,6 @@ func (t *EmbedImages_Image) UnmarshalCBOR(r io.Reader) (err error) { } } - // t.AspectRatio (bsky.EmbedImages_AspectRatio) (struct) - case "aspectRatio": - - { - - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - t.AspectRatio = new(EmbedImages_AspectRatio) - if err := t.AspectRatio.UnmarshalCBOR(cr); err != nil { - return xerrors.Errorf("unmarshaling t.AspectRatio pointer: %w", err) - } - } - - } default: // Field doesn't exist on this type, so ignore it diff --git a/api/bsky/embedimages.go b/api/bsky/embedimages.go index 06a6ca6f..b0cdb8a6 100644 --- a/api/bsky/embedimages.go +++ b/api/bsky/embedimages.go @@ -17,18 +17,9 @@ type EmbedImages struct { Images []*EmbedImages_Image `json:"images" cborgen:"images"` } -// EmbedImages_AspectRatio is a "aspectRatio" in the app.bsky.embed.images schema. -// -// width:height represents an aspect ratio. It may be approximate, and may not correspond to absolute dimensions in any given unit. -type EmbedImages_AspectRatio struct { - Height int64 `json:"height" cborgen:"height"` - Width int64 `json:"width" cborgen:"width"` -} - // EmbedImages_Image is a "image" in the app.bsky.embed.images schema. type EmbedImages_Image struct { Alt string `json:"alt" cborgen:"alt"` - AspectRatio *EmbedImages_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` Image *util.LexBlob `json:"image" cborgen:"image"` } @@ -43,7 +34,6 @@ type EmbedImages_View struct { // EmbedImages_ViewImage is a "viewImage" in the app.bsky.embed.images schema. type EmbedImages_ViewImage struct { Alt string `json:"alt" cborgen:"alt"` - AspectRatio *EmbedImages_AspectRatio `json:"aspectRatio,omitempty" cborgen:"aspectRatio,omitempty"` Fullsize string `json:"fullsize" cborgen:"fullsize"` Thumb string `json:"thumb" cborgen:"thumb"` } diff --git a/api/bsky/feedthreadgate.go b/api/bsky/feedthreadgate.go deleted file mode 100644 index 14d45393..00000000 --- a/api/bsky/feedthreadgate.go +++ /dev/null @@ -1,128 +0,0 @@ -// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. - -package bsky - -// schema: app.bsky.feed.threadgate - -import ( - "bytes" - "encoding/json" - "fmt" - "io" - - "github.com/bluesky-social/indigo/lex/util" - cbg "github.com/whyrusleeping/cbor-gen" -) - -func init() { - util.RegisterType("app.bsky.feed.threadgate", &FeedThreadgate{}) -} // -// RECORDTYPE: FeedThreadgate -type FeedThreadgate struct { - LexiconTypeID string `json:"$type,const=app.bsky.feed.threadgate" cborgen:"$type,const=app.bsky.feed.threadgate"` - Allow []*FeedThreadgate_Allow_Elem `json:"allow,omitempty" cborgen:"allow,omitempty"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Post string `json:"post" cborgen:"post"` -} - -type FeedThreadgate_Allow_Elem struct { - FeedThreadgate_MentionRule *FeedThreadgate_MentionRule - FeedThreadgate_FollowingRule *FeedThreadgate_FollowingRule - FeedThreadgate_ListRule *FeedThreadgate_ListRule -} - -func (t *FeedThreadgate_Allow_Elem) MarshalJSON() ([]byte, error) { - if t.FeedThreadgate_MentionRule != nil { - t.FeedThreadgate_MentionRule.LexiconTypeID = "app.bsky.feed.threadgate#mentionRule" - return json.Marshal(t.FeedThreadgate_MentionRule) - } - if t.FeedThreadgate_FollowingRule != nil { - t.FeedThreadgate_FollowingRule.LexiconTypeID = "app.bsky.feed.threadgate#followingRule" - return json.Marshal(t.FeedThreadgate_FollowingRule) - } - if t.FeedThreadgate_ListRule != nil { - t.FeedThreadgate_ListRule.LexiconTypeID = "app.bsky.feed.threadgate#listRule" - return json.Marshal(t.FeedThreadgate_ListRule) - } - return nil, fmt.Errorf("cannot marshal empty enum") -} -func (t *FeedThreadgate_Allow_Elem) UnmarshalJSON(b []byte) error { - typ, err := util.TypeExtract(b) - if err != nil { - return err - } - - switch typ { - case "app.bsky.feed.threadgate#mentionRule": - t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) - return json.Unmarshal(b, t.FeedThreadgate_MentionRule) - case "app.bsky.feed.threadgate#followingRule": - t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) - return json.Unmarshal(b, t.FeedThreadgate_FollowingRule) - case "app.bsky.feed.threadgate#listRule": - t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) - return json.Unmarshal(b, t.FeedThreadgate_ListRule) - - default: - return nil - } -} - -func (t *FeedThreadgate_Allow_Elem) MarshalCBOR(w io.Writer) error { - - if t == nil { - _, err := w.Write(cbg.CborNull) - return err - } - if t.FeedThreadgate_MentionRule != nil { - return t.FeedThreadgate_MentionRule.MarshalCBOR(w) - } - if t.FeedThreadgate_FollowingRule != nil { - return t.FeedThreadgate_FollowingRule.MarshalCBOR(w) - } - if t.FeedThreadgate_ListRule != nil { - return t.FeedThreadgate_ListRule.MarshalCBOR(w) - } - return fmt.Errorf("cannot cbor marshal empty enum") -} -func (t *FeedThreadgate_Allow_Elem) UnmarshalCBOR(r io.Reader) error { - typ, b, err := util.CborTypeExtractReader(r) - if err != nil { - return err - } - - switch typ { - case "app.bsky.feed.threadgate#mentionRule": - t.FeedThreadgate_MentionRule = new(FeedThreadgate_MentionRule) - return t.FeedThreadgate_MentionRule.UnmarshalCBOR(bytes.NewReader(b)) - case "app.bsky.feed.threadgate#followingRule": - t.FeedThreadgate_FollowingRule = new(FeedThreadgate_FollowingRule) - return t.FeedThreadgate_FollowingRule.UnmarshalCBOR(bytes.NewReader(b)) - case "app.bsky.feed.threadgate#listRule": - t.FeedThreadgate_ListRule = new(FeedThreadgate_ListRule) - return t.FeedThreadgate_ListRule.UnmarshalCBOR(bytes.NewReader(b)) - - default: - return nil - } -} - -// FeedThreadgate_FollowingRule is a "followingRule" in the app.bsky.feed.threadgate schema. -// -// Allow replies from actors you follow. -type FeedThreadgate_FollowingRule interface{} - -// FeedThreadgate_ListRule is a "listRule" in the app.bsky.feed.threadgate schema. -// -// Allow replies from actors on a list. -// -// RECORDTYPE: FeedThreadgate_ListRule -type FeedThreadgate_ListRule struct { - LexiconTypeID string `json:"$type,const=app.bsky.feed.threadgate#listRule" cborgen:"$type,const=app.bsky.feed.threadgate#listRule"` - List string `json:"list" cborgen:"list"` -} - -// FeedThreadgate_MentionRule is a "mentionRule" in the app.bsky.feed.threadgate schema. -// -// Allow replies from actors mentioned in your post. -type FeedThreadgate_MentionRule interface{} diff --git a/api/bsky/graphlistblock.go b/api/bsky/graphlistblock.go deleted file mode 100644 index ee743007..00000000 --- a/api/bsky/graphlistblock.go +++ /dev/null @@ -1,19 +0,0 @@ -// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. - -package bsky - -// schema: app.bsky.graph.listblock - -import ( - "github.com/bluesky-social/indigo/lex/util" -) - -func init() { - util.RegisterType("app.bsky.graph.listblock", &GraphListblock{}) -} // -// RECORDTYPE: GraphListblock -type GraphListblock struct { - LexiconTypeID string `json:"$type,const=app.bsky.graph.listblock" cborgen:"$type,const=app.bsky.graph.listblock"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Subject string `json:"subject" cborgen:"subject"` -} -- 2.51.2 From 59cb6d8004b31ccf01debdf26149700bec80e5cd Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 5 Oct 2023 19:49:49 +0000 Subject: [PATCH 115/192] Track repo compaction duration and log progress every once in a while --- bgs/bgs.go | 14 +++++++++++++- bgs/metrics.go | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index a5e04d53..a28e6736 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1211,6 +1211,10 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction") defer span.End() + log.Warn("starting repo compaction") + + runStart := time.Now() + repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50) if err != nil { return nil, fmt.Errorf("failed to get repos to compact: %w", err) @@ -1227,7 +1231,7 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp } results := make(map[models.Uid]*carstore.CompactionStats) - for _, r := range repos { + for i, r := range repos { select { case <-ctx.Done(): return &compactionStats{ @@ -1237,14 +1241,22 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp default: } + repostart := time.Now() st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr) if err != nil { log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) continue } + compactionDuration.Observe(time.Since(repostart).Seconds()) results[r.Usr] = st + + if i%100 == 0 { + log.Warnf("compacted %d repos in %s", i+1, time.Since(runStart)) + } } + log.Warnf("compacted %d repos in %s", len(repos), time.Since(runStart)) + return &compactionStats{ Targets: repos, Completed: results, diff --git a/bgs/metrics.go b/bgs/metrics.go index 97f444c4..12b401e0 100644 --- a/bgs/metrics.go +++ b/bgs/metrics.go @@ -42,6 +42,12 @@ var externalUserCreationAttempts = promauto.NewCounter(prometheus.CounterOpts{ Help: "The total number of external users created", }) +var compactionDuration = promauto.NewHistogram(prometheus.HistogramOpts{ + Name: "compaction_duration", + Help: "A histogram of compaction latencies", + Buckets: prometheus.ExponentialBuckets(0.001, 3, 14), +}) + var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_new_users_discovered", Help: "The total number of new users discovered directly from the firehose (not from refs)", -- 2.51.2 From aa141d31b8ebf903173991b9c8eb1b0aada8e46a Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Thu, 5 Oct 2023 17:27:09 -0700 Subject: [PATCH 116/192] syntax: constant for 'invalid.handle' --- atproto/syntax/handle.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/atproto/syntax/handle.go b/atproto/syntax/handle.go index 73b8f622..9693f8b0 100644 --- a/atproto/syntax/handle.go +++ b/atproto/syntax/handle.go @@ -6,7 +6,12 @@ import ( "strings" ) -var handleRegex = regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`) +var ( + handleRegex = regexp.MustCompile(`^([a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?\.)+[a-zA-Z]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$`) + + // special handle string constant indicating that handle resolution failed + HandleInvalid = Handle("handle.invalid") +) // String type which represents a syntaxtually valid handle identifier, as would pass Lexicon syntax validation. // -- 2.51.2 From 209acd916640c12fa7a6d3ec4694b34e043fef4e Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Thu, 5 Oct 2023 17:27:25 -0700 Subject: [PATCH 117/192] identity: use HandleInvalid constant --- atproto/identity/base_directory.go | 2 +- atproto/identity/live_test.go | 2 +- atproto/identity/mock_directory_test.go | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/atproto/identity/base_directory.go b/atproto/identity/base_directory.go index d53a0bb0..6b8c8325 100644 --- a/atproto/identity/base_directory.go +++ b/atproto/identity/base_directory.go @@ -73,7 +73,7 @@ func (d *BaseDirectory) LookupDID(ctx context.Context, did syntax.DID) (*Identit if err != nil && err != ErrHandleNotFound { return nil, err } else if ErrHandleNotFound == err || resolvedDID != did { - ident.Handle = syntax.Handle("handle.invalid") + ident.Handle = syntax.HandleInvalid } else { ident.Handle = declared } diff --git a/atproto/identity/live_test.go b/atproto/identity/live_test.go index c06213d2..60bb160e 100644 --- a/atproto/identity/live_test.go +++ b/atproto/identity/live_test.go @@ -50,7 +50,7 @@ func testDirectoryLive(t *testing.T, d Directory) { _, err = d.LookupDID(ctx, syntax.DID("did:plc:fake-dummy-no-resolve.atproto.com")) assert.Equal(ErrDIDNotFound, err) - _, err = d.LookupHandle(ctx, syntax.Handle("handle.invalid")) + _, err = d.LookupHandle(ctx, syntax.HandleInvalid) assert.Error(err) } diff --git a/atproto/identity/mock_directory_test.go b/atproto/identity/mock_directory_test.go index 139c187a..c916e8a5 100644 --- a/atproto/identity/mock_directory_test.go +++ b/atproto/identity/mock_directory_test.go @@ -20,7 +20,7 @@ func TestMockDirectory(t *testing.T) { } id2 := Identity{ DID: syntax.DID("did:plc:abc222"), - Handle: syntax.Handle("handle.invalid"), + Handle: syntax.HandleInvalid, } id3 := Identity{ DID: syntax.DID("did:plc:abc333"), @@ -48,7 +48,7 @@ func TestMockDirectory(t *testing.T) { assert.NoError(err) assert.True(out.Handle.IsInvalidHandle()) - _, err = c.LookupHandle(ctx, syntax.Handle("handle.invalid")) + _, err = c.LookupHandle(ctx, syntax.HandleInvalid) assert.Equal(ErrHandleNotFound, err) out, err = c.LookupDID(ctx, syntax.DID("did:plc:abc999")) assert.Equal(ErrDIDNotFound, err) -- 2.51.2 From aa95123bf6bd1516d46ba5e0854a61a9760ae76c Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Thu, 5 Oct 2023 17:42:37 -0700 Subject: [PATCH 118/192] syntax: disallow .example and .alt TLDs --- atproto/syntax/handle.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/atproto/syntax/handle.go b/atproto/syntax/handle.go index 9693f8b0..0efe2917 100644 --- a/atproto/syntax/handle.go +++ b/atproto/syntax/handle.go @@ -31,6 +31,8 @@ func ParseHandle(raw string) (Handle, error) { } // Some top-level domains (TLDs) are disallowed for registration across the atproto ecosystem. The *syntax* is valid, but these should never be considered acceptable handles for account registration or linking. +// +// The reserved '.test' TLD is allowed, for testing and development. It is expected that '.test' domain resolution will fail in a real-world network. func (h Handle) AllowedTLD() bool { switch h.TLD() { case "local", @@ -38,7 +40,9 @@ func (h Handle) AllowedTLD() bool { "invalid", "localhost", "internal", - "onion": + "example", + "onion", + "alt": return false } return true -- 2.51.2 From 56e82da2aa392b4b6d5d1c1a4ae6d8c8e742bb9f Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Thu, 5 Oct 2023 17:43:06 -0700 Subject: [PATCH 119/192] identity: enforce disallowed TLDs --- atproto/identity/handle.go | 4 ++++ atproto/identity/identity.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/atproto/identity/handle.go b/atproto/identity/handle.go index dfa3619b..afbecd78 100644 --- a/atproto/identity/handle.go +++ b/atproto/identity/handle.go @@ -164,6 +164,10 @@ func (d *BaseDirectory) ResolveHandle(ctx context.Context, handle syntax.Handle) var dnsErr error var did syntax.DID + if !handle.AllowedTLD() { + return "", ErrHandleReservedTLD + } + tryDNS := true for _, suffix := range d.SkipDNSDomainSuffixes { if strings.HasSuffix(handle.String(), suffix) { diff --git a/atproto/identity/identity.go b/atproto/identity/identity.go index 1e88e72a..a9d2a9a5 100644 --- a/atproto/identity/identity.go +++ b/atproto/identity/identity.go @@ -40,6 +40,9 @@ var ErrHandleNotFound = errors.New("handle not found") // Indicates that handle and DID resolved, but handle points to a DID with a different handle. This is only returned when looking up a handle, not when looking up a DID. var ErrHandleNotValid = errors.New("handle resolves to DID with different handle") +// Handle top-level domain (TLD) is one of the special "Reserved" suffixes, and not allowed for atproto use +var ErrHandleReservedTLD = errors.New("handle top-level domain is disallowed") + // Indicates that resolution process completed successfully, but the DID does not exist. var ErrDIDNotFound = errors.New("DID not found") -- 2.51.2 From 71a5af60d26c8257d031d81c1e2838861205eeb0 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 6 Oct 2023 12:14:36 -0700 Subject: [PATCH 120/192] add flag to skip big shards during compaction --- bgs/admin.go | 14 ++++++++++++-- bgs/bgs.go | 4 ++-- carstore/bs.go | 34 +++++++++++++++++++++++++++++++++- carstore/repo_test.go | 4 ++-- cmd/gosky/bgs.go | 33 ++++++++++++++++++++++++++++----- 5 files changed, 77 insertions(+), 12 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index 4044c9c1..d83ae5bb 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -406,12 +406,17 @@ func (bgs *BGS) handleAdminCompactRepo(e echo.Context) error { return fmt.Errorf("must pass a did") } + var fast bool + if strings.ToLower(e.QueryParam("fast")) == "true" { + fast = true + } + u, err := bgs.lookupUserByDid(ctx, did) if err != nil { return fmt.Errorf("no such user: %w", err) } - stats, err := bgs.repoman.CarStore().CompactUserShards(ctx, u.ID) + stats, err := bgs.repoman.CarStore().CompactUserShards(ctx, u.ID, fast) if err != nil { return fmt.Errorf("compaction failed: %w", err) } @@ -431,6 +436,11 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { dry = true } + var fast bool + if strings.ToLower(e.QueryParam("fast")) == "true" { + fast = true + } + lim := 50 if limstr := e.QueryParam("limit"); limstr != "" { v, err := strconv.Atoi(limstr) @@ -441,7 +451,7 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { lim = v } - stats, err := bgs.runRepoCompaction(ctx, lim, dry) + stats, err := bgs.runRepoCompaction(ctx, lim, dry, fast) if err != nil { return fmt.Errorf("compaction run failed: %w", err) } diff --git a/bgs/bgs.go b/bgs/bgs.go index e519ae97..30eee5df 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1207,7 +1207,7 @@ type compactionStats struct { Targets []carstore.CompactionTarget } -func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*compactionStats, error) { +func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool, fast bool) (*compactionStats, error) { ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction") defer span.End() @@ -1242,7 +1242,7 @@ func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool) (*comp } repostart := time.Now() - st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr) + st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr, fast) if err != nil { log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) continue diff --git a/carstore/bs.go b/carstore/bs.go index 56839556..1a7cd94a 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -36,6 +36,8 @@ var log = logging.Logger("carstore") const MaxSliceLength = 2 << 20 +const BigShardThreshold = 2 << 20 + type CarStore struct { meta *gorm.DB rootDir string @@ -1207,6 +1209,15 @@ func (cs *CarStore) getBlockRefsForShards(ctx context.Context, shardIds []uint) return out, nil } +func shardSize(sh *CarShard) (int64, error) { + st, err := os.Stat(sh.Path) + if err != nil { + return 0, fmt.Errorf("stat %q: %w", sh.Path, err) + } + + return st.Size(), nil +} + type CompactionStats struct { TotalRefs int `json:"totalRefs"` StartShards int `json:"startShards"` @@ -1217,7 +1228,7 @@ type CompactionStats struct { DupeCount int `json:"dupeCount"` } -func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*CompactionStats, error) { +func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skipBigShards bool) (*CompactionStats, error) { ctx, span := otel.Tracer("carstore").Start(ctx, "CompactUserShards") defer span.End() @@ -1228,6 +1239,27 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co return nil, err } + sort.Slice(shards, func(i, j int) bool { + return shards[i].Seq < shards[j].Seq + }) + + if skipBigShards { + var skip int + for i, sh := range shards { + size, err := shardSize(&sh) + if err != nil { + return nil, fmt.Errorf("could not check size of shard file: %w", err) + } + + if size > BigShardThreshold { + skip = i + 1 + } else { + break + } + } + shards = shards[skip:] + } + span.SetAttributes(attribute.Int("shards", len(shards))) var shardIds []uint diff --git a/carstore/repo_test.go b/carstore/repo_test.go index 1093f459..0c6e9b0a 100644 --- a/carstore/repo_test.go +++ b/carstore/repo_test.go @@ -142,7 +142,7 @@ func TestBasicOperation(t *testing.T) { } checkRepo(t, cs, buf, recs) - if _, err := cs.CompactUserShards(ctx, 1); err != nil { + if _, err := cs.CompactUserShards(ctx, 1, false); err != nil { t.Fatal(err) } @@ -228,7 +228,7 @@ func TestRepeatedCompactions(t *testing.T) { head = nroot } fmt.Println("Run compaction", loop) - st, err := cs.CompactUserShards(ctx, 1) + st, err := cs.CompactUserShards(ctx, 1, false) if err != nil { t.Fatal(err) } diff --git a/cmd/gosky/bgs.go b/cmd/gosky/bgs.go index 6764a304..7806113a 100644 --- a/cmd/gosky/bgs.go +++ b/cmd/gosky/bgs.go @@ -305,14 +305,28 @@ var bgsSetNewSubsEnabledCmd = &cli.Command{ var bgsCompactRepo = &cli.Command{ Name: "compact-repo", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "fast", + }, + }, Action: func(cctx *cli.Context) error { - url := cctx.String("bgs") + "/admin/repo/compact" + uu, err := url.Parse(cctx.String("bgs") + "/admin/repo/compact") + if err != nil { + return err + } + q := uu.Query() did := cctx.Args().First() + q.Add("did", did) - url += fmt.Sprintf("?did=%s", did) + if cctx.Bool("fast") { + q.Add("fast", "true") + } - req, err := http.NewRequest("POST", url, nil) + uu.RawQuery = q.Encode() + + req, err := http.NewRequest("POST", uu.String(), nil) if err != nil { return err } @@ -354,6 +368,9 @@ var bgsCompactAll = &cli.Command{ &cli.IntFlag{ Name: "limit", }, + &cli.BoolFlag{ + Name: "fast", + }, }, Action: func(cctx *cli.Context) error { uu, err := url.Parse(cctx.String("bgs") + "/admin/repo/compactAll") @@ -361,13 +378,19 @@ var bgsCompactAll = &cli.Command{ return err } + q := uu.Query() if cctx.Bool("dry") { - uu.Query().Add("dry", "true") + q.Add("dry", "true") + } + + if cctx.Bool("fast") { + q.Add("fast", "true") } if cctx.IsSet("limit") { - uu.Query().Add("limit", fmt.Sprint(cctx.Int("limit"))) + q.Add("limit", fmt.Sprint(cctx.Int("limit"))) } + uu.RawQuery = q.Encode() req, err := http.NewRequest("POST", uu.String(), nil) if err != nil { -- 2.51.2 From cf599354ce23839b825f987d93dbacee3e1ae34e Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 6 Oct 2023 12:18:39 -0700 Subject: [PATCH 121/192] use cli context instead of TODO --- cmd/gosky/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/gosky/main.go b/cmd/gosky/main.go index 8b240b17..528440db 100644 --- a/cmd/gosky/main.go +++ b/cmd/gosky/main.go @@ -1576,7 +1576,7 @@ var requestAccountDeletionCmd = &cli.Command{ return err } - err = comatproto.ServerRequestAccountDelete(context.TODO(), xrpcc) + err = comatproto.ServerRequestAccountDelete(cctx.Context, xrpcc) if err != nil { return err } @@ -1596,7 +1596,7 @@ var deleteAccountCmd = &cli.Command{ token := cctx.Args().First() password := cctx.Args().Get(1) - err = comatproto.ServerDeleteAccount(context.TODO(), xrpcc, &comatproto.ServerDeleteAccount_Input{ + err = comatproto.ServerDeleteAccount(cctx.Context, xrpcc, &comatproto.ServerDeleteAccount_Input{ Did: xrpcc.Auth.Did, Token: token, Password: password, -- 2.51.2 From 7a4c06a7d762212364f41720c9f8623532e2417d Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 18:29:46 -0700 Subject: [PATCH 122/192] import old cmd/pubweb work as cmd/athome --- cmd/athome/README.md | 14 ++ cmd/athome/main.go | 72 ++++++++ cmd/athome/renderer.go | 85 ++++++++++ cmd/athome/server.go | 218 +++++++++++++++++++++++++ cmd/athome/static/apple-touch-icon.png | Bin 0 -> 14742 bytes cmd/athome/static/default-avatar.png | Bin 0 -> 1624 bytes cmd/athome/static/favicon-16x16.png | Bin 0 -> 316 bytes cmd/athome/static/favicon-32x32.png | Bin 0 -> 769 bytes cmd/athome/static/favicon.ico | Bin 0 -> 15406 bytes cmd/athome/static/favicon.png | Bin 0 -> 6405 bytes cmd/athome/static/robots.txt | 9 + cmd/athome/templates/base.html | 55 +++++++ cmd/athome/templates/error.html | 12 ++ cmd/athome/templates/feed_macros.html | 106 ++++++++++++ cmd/athome/templates/home.html | 20 +++ cmd/athome/templates/post.html | 46 ++++++ cmd/athome/templates/profile.html | 67 ++++++++ 17 files changed, 704 insertions(+) create mode 100644 cmd/athome/README.md create mode 100644 cmd/athome/main.go create mode 100644 cmd/athome/renderer.go create mode 100644 cmd/athome/server.go create mode 100644 cmd/athome/static/apple-touch-icon.png create mode 100644 cmd/athome/static/default-avatar.png create mode 100644 cmd/athome/static/favicon-16x16.png create mode 100644 cmd/athome/static/favicon-32x32.png create mode 100644 cmd/athome/static/favicon.ico create mode 100644 cmd/athome/static/favicon.png create mode 100644 cmd/athome/static/robots.txt create mode 100644 cmd/athome/templates/base.html create mode 100644 cmd/athome/templates/error.html create mode 100644 cmd/athome/templates/feed_macros.html create mode 100644 cmd/athome/templates/home.html create mode 100644 cmd/athome/templates/post.html create mode 100644 cmd/athome/templates/profile.html diff --git a/cmd/athome/README.md b/cmd/athome/README.md new file mode 100644 index 00000000..6bb954b2 --- /dev/null +++ b/cmd/athome/README.md @@ -0,0 +1,14 @@ + +athome: Public Bluesky Web Home +=============================== + +```text +me: can we have public web interface? +mom: we have public web interface at home +public web interface at home: +``` + +1. run this web service somewhere +2. point one or more handle domains to it (CNAME or reverse proxy) +3. serves up profile and feed for that account only +4. fetches data from public bsky app view API diff --git a/cmd/athome/main.go b/cmd/athome/main.go new file mode 100644 index 00000000..084aa70a --- /dev/null +++ b/cmd/athome/main.go @@ -0,0 +1,72 @@ +package main + +import ( + "os" + + _ "github.com/joho/godotenv/autoload" + + logging "github.com/ipfs/go-log" + "github.com/urfave/cli/v2" +) + +var log = logging.Logger("pubweb") + +func init() { + logging.SetAllLoggers(logging.LevelDebug) + //logging.SetAllLoggers(logging.LevelWarn) +} + +func main() { + run(os.Args) +} + +func run(args []string) { + + app := cli.App{ + Name: "pubweb", + Usage: "public web interface to bluesky content", + } + + app.Commands = []*cli.Command{ + &cli.Command{ + Name: "serve", + Usage: "run the server", + Action: serve, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "pds-host", + Usage: "method, hostname, and port of PDS instance", + Value: "http://localhost:4849", + EnvVars: []string{"ATP_PDS_HOST"}, + }, + &cli.StringFlag{ + Name: "handle", + Usage: "for PDS login", + Required: true, + EnvVars: []string{"ATP_AUTH_HANDLE"}, + }, + &cli.StringFlag{ + Name: "password", + Usage: "for PDS login", + Required: true, + EnvVars: []string{"ATP_AUTH_PASSWORD"}, + }, + &cli.StringFlag{ + Name: "http-address", + Usage: "Specify the local IP/port to bind to", + Required: false, + Value: ":8200", + EnvVars: []string{"PUBWEB_BIND"}, + }, + &cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug mode", + Value: false, + Required: false, + EnvVars: []string{"DEBUG"}, + }, + }, + }, + } + app.RunAndExitOnError() +} diff --git a/cmd/athome/renderer.go b/cmd/athome/renderer.go new file mode 100644 index 00000000..0fa3cb1c --- /dev/null +++ b/cmd/athome/renderer.go @@ -0,0 +1,85 @@ +package main + +import ( + "bytes" + "embed" + "errors" + "fmt" + "io" + "path/filepath" + + "github.com/flosch/pongo2/v6" + "github.com/labstack/echo/v4" +) + +//go:embed templates/* +var TemplateFS embed.FS + +type RendererLoader struct { + prefix string + fs *embed.FS +} + +func NewRendererLoader(prefix string, fs *embed.FS) pongo2.TemplateLoader { + return &RendererLoader{ + prefix: prefix, + fs: fs, + } +} +func (l *RendererLoader) Abs(_, name string) string { + // TODO: remove this workaround + // Figure out why this method is being called + // twice on template names resulting in a failure to resolve + // the template name. + if filepath.HasPrefix(name, l.prefix) { + return name + } + return filepath.Join(l.prefix, name) +} + +func (l *RendererLoader) Get(path string) (io.Reader, error) { + b, err := l.fs.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("reading template %q failed: %w", path, err) + } + return bytes.NewReader(b), nil +} + +type Renderer struct { + TemplateSet *pongo2.TemplateSet + Debug bool +} + +func NewRenderer(prefix string, fs *embed.FS, debug bool) *Renderer { + return &Renderer{ + TemplateSet: pongo2.NewSet(prefix, NewRendererLoader(prefix, fs)), + Debug: debug, + } +} + +func (r Renderer) Render(w io.Writer, name string, data interface{}, c echo.Context) error { + var ctx pongo2.Context + + if data != nil { + var ok bool + ctx, ok = data.(pongo2.Context) + if !ok { + return errors.New("no pongo2.Context data was passed") + } + } + + var t *pongo2.Template + var err error + + if r.Debug { + t, err = pongo2.FromFile(name) + } else { + t, err = r.TemplateSet.FromFile(name) + } + + if err != nil { + return err + } + + return t.ExecuteWriter(ctx, w) +} diff --git a/cmd/athome/server.go b/cmd/athome/server.go new file mode 100644 index 00000000..49dd23d7 --- /dev/null +++ b/cmd/athome/server.go @@ -0,0 +1,218 @@ +package main + +import ( + "context" + "fmt" + "io/fs" + "net/http" + "os" + "strings" + "embed" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + appbsky "github.com/bluesky-social/indigo/api/bsky" + cliutil "github.com/bluesky-social/indigo/cmd/gosky/util" + "github.com/bluesky-social/indigo/xrpc" + + "github.com/flosch/pongo2/v6" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/urfave/cli/v2" +) + +//go:embed static/* +var StaticFS embed.FS + +type Server struct { + xrpcc *xrpc.Client +} + +func serve(cctx *cli.Context) error { + debug := cctx.Bool("debug") + httpAddress := cctx.String("http-address") + pdsHost := cctx.String("pds-host") + atpHandle := cctx.String("handle") + atpPassword := cctx.String("password") + + // create a new session + // TODO: does this work with no auth at all? + xrpcc := &xrpc.Client{ + Client: cliutil.NewHttpClient(), + Host: pdsHost, + Auth: &xrpc.AuthInfo{ + Handle: atpHandle, + }, + } + + auth, err := comatproto.ServerCreateSession(context.TODO(), xrpcc, &comatproto.ServerCreateSession_Input{ + Identifier: xrpcc.Auth.Handle, + Password: atpPassword, + }) + if err != nil { + return err + } + xrpcc.Auth.AccessJwt = auth.AccessJwt + xrpcc.Auth.RefreshJwt = auth.RefreshJwt + xrpcc.Auth.Did = auth.Did + xrpcc.Auth.Handle = auth.Handle + + server := Server{xrpcc} + + staticHandler := http.FileServer(func() http.FileSystem { + if debug { + return http.FS(os.DirFS("static")) + } + fsys, err := fs.Sub(StaticFS, "static") + if err != nil { + log.Fatal(err) + } + return http.FS(fsys) + }()) + + e := echo.New() + e.HideBanner = true + // SECURITY: Do not modify without due consideration. + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + ContentTypeNosniff: "nosniff", + XFrameOptions: "SAMEORIGIN", + HSTSMaxAge: 31536000, // 365 days + // TODO: + // ContentSecurityPolicy + // XSSProtection + })) + e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ + // Don't log requests for static content. + Skipper: func(c echo.Context) bool { + return strings.HasPrefix(c.Request().URL.Path, "/static") + }, + Format: "method=${method} path=${uri} status=${status} latency=${latency_human}\n", + })) + e.Renderer = NewRenderer("templates/", &TemplateFS, debug) + e.HTTPErrorHandler = customHTTPErrorHandler + + // redirect trailing slash to non-trailing slash. + // all of our current endpoints have no trailing slash. + e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{ + RedirectCode: http.StatusFound, + })) + + // basic static routes + e.GET("/robots.txt", echo.WrapHandler(staticHandler)) + e.GET("/favicon.ico", echo.WrapHandler(staticHandler)) + e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler))) + e.GET("/", server.WebHome) + + // generic routes + //e.GET("/search", server.WebGeneric) + //e.GET("/support", server.WebGeneric) + //e.GET("/support/privacy", server.WebGeneric) + //e.GET("/support/tos", server.WebGeneric) + //e.GET("/support/community-guidelines", server.WebGeneric) + //e.GET("/support/copyright", server.WebGeneric) + + // profile endpoints; only first populates info + e.GET("/profile/:handle", server.WebProfile) + //e.GET("/profile/:handle/repo.car.gz", server.WebProfile) + //e.GET("/profile/:handle/follows", server.WebGeneric) + //e.GET("/profile/:handle/followers", server.WebGeneric) + + // post endpoints; only first populates info + e.GET("/profile/:handle/post/:rkey", server.WebPost) + //e.GET("/profile/:handle/post/:rkey/liked-by", server.WebGeneric) + //e.GET("/profile/:handle/post/:rkey/reposted-by", server.WebGeneric) + + // feeds + e.GET("/feed/:name", server.WebFeed) + + // redirect + //e.GET("/at://:account", server.WebAccountURI) + //e.GET("/at://:account/:nsid/:rkey", server.WebRecordURI) + + log.Infow("starting server", "bind", httpAddress) + return e.Start(httpAddress) +} + +func customHTTPErrorHandler(err error, c echo.Context) { + code := http.StatusInternalServerError + if he, ok := err.(*echo.HTTPError); ok { + code = he.Code + } + c.Logger().Error(err) + data := pongo2.Context{ + "statusCode": code, + } + c.Render(code, "error.html", data) +} + +func (srv *Server) WebHome(c echo.Context) error { + data := pongo2.Context{} + return c.Render(http.StatusOK, "home.html", data) +} + +func (srv *Server) WebPost(c echo.Context) error { + data := pongo2.Context{} + handle := c.Param("handle") + rkey := c.Param("rkey") + // sanity check argument + if len(handle) > 4 && len(handle) < 128 && len(rkey) > 0 { + ctx := c.Request().Context() + // requires two fetches: first fetch profile (!) + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) + if err != nil { + log.Warnf("failed to fetch handle: %s\t%v", handle, err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) + } else { + did := pv.Did + data["did"] = did + + // then fetch the post thread (with extra context) + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) + tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, uri) + if err != nil { + log.Warnf("failed to fetch post: %s\t%v", uri, err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "post not found: %s", handle) + } else { + req := c.Request() + postView := tpv.Thread.FeedDefs_ThreadViewPost.Post + data["postView"] = postView + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + if postView.Embed != nil && postView.Embed.EmbedImages_View != nil { + data["imgThumbUrl"] = postView.Embed.EmbedImages_View.Images[0].Thumb + } + } + } + + } + return c.Render(http.StatusOK, "post.html", data) +} + +func (srv *Server) WebProfile(c echo.Context) error { + data := pongo2.Context{} + handle := c.Param("handle") + // sanity check argument + if len(handle) > 4 && len(handle) < 128 { + ctx := c.Request().Context() + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) + if err != nil { + log.Warnw("failed to fetch handle", "handle", handle, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) + } else { + req := c.Request() + data["profileView"] = pv + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + } + af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle, "", 100) + if err != nil { + log.Warnw("failed to fetch author feed", "handle", handle, "err", err) + // TODO: show some error? + } else { + data["authorFeed"] = af.Feed + //log.Warnw("author feed", "feed", af.Feed) + } + } + + return c.Render(http.StatusOK, "profile.html", data) +} diff --git a/cmd/athome/static/apple-touch-icon.png b/cmd/athome/static/apple-touch-icon.png new file mode 100644 index 0000000000000000000000000000000000000000..5ebb6787c999108d56d1b2f2aa4a5f2275ac5e37 GIT binary patch literal 14742 zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}9Bd2>47O+4j2IXc7(87ZLn`9l*4})d8(w?g z&OZJ(U(9*4%^Nw5+a`!i6cL!Xbjj3NF13$W>esFax+E55wY2!ITBh?Vv!JDRA#dv^ zOz#qL&5}AQ+Bs3d&&wu(i><$c$%g;?uV3@(_GPc%dcJ(^*SXj1p3jm#a{gZV{LSB^ z?tlC&`>n$4^1*-G+%Kyy55D|*+2zY0U(R}Y=gYR28M`vA?DvZ5+?z0c_42bNqIU6y zxyvU<^gaGvSYz*XTI@92|Iqr|3tG-@G2t%${AG*y|063jw}^O}-LCx8x9zCwBH56Q zJkqKYk6cVK>R1|kY|q_w;fc3DOVr36KL6meWKEsq>+}zEr!D{R_v02BU;gCEf?XxM z>Z=le-TPAe=Xa&f<+f+<|C(n>9W$sbG_I}KmGW!jO9{JpwvKsi8>T0p-Wsu=_1E59 ztNJf$Z>nvl?2>!Edc)iUHI`}fJQ|i&%3KI`V_&%5bVW+EKeOF&>D?LOY{wq#@7Qz3 zo@D{Al(colWQo%ggpMjVJd0G&4qaAWa`~U^g5@uB*i0IKSvvfYT$5QTbbEfodHJ&o z>NsxigGj=FO!0X}g48Yk&A#`lE)+Yu=IP-k+Tg8l*qV-R_xuYSZ%#dzkX~HGTEI@k_{Vf5X?! zmdW<5U(1DOn4h>HT=eDV9=mp*eof=i0UK{8lo0n|iso z;jwjawV`kRxzk%eyzu1Ud64%^$7Z`Y%e~}G-?v9ue&h>J@jk1jojS8A^3vr5Ujdz$ zx%>{3kGkBOci`rg+YZy#duTek+}x9ExvKKb=|huJGfX#zyF4pvblBVYAe!l0o6%d# z8J}eKUEXy1jyr>8V=dd~#W%il9J||pr$*lHK7Wagb*<>m-+jESM=!)**fsN+?(O7; z7pf;tvOLjNJn}W-Uh~s5o%d}9SM)DCMN0l>oWpV>h#~!P+J|k^q8Mrf8LCdSnHEHM zXJu9CC~bW5QAtkbLSX*qZ%S-0EbY@n!dK-6tef`WNV?OJ9K&CKOd2Jlu1Cnx%w3h$k_y$9yPjxY(AmR!Cf_Tv()upL!Fz1S z@{c&>vN!%LU0Lg~S8nz3rQfqR_h*WdM za4d7DuHVU}&2};m82_9;c(z4wd1b%Al|k96ht>vPvL%&6xvzTfz?R^iv=2xaU1Lb2vp5l?oxH&ddk zrB)~Breu7S$yO^+G}1`Oz81u{uAkrp{$~IyY+V zAN@O}B1+yi?P+aI3NO-JJ^rLUI;N>iV@QC@alqK5wPqY4Z(t zy7RxQ70h>*{>1d_)$@xjfAX^Ik8HRXxFonZ-+TF!bxbqz5B*ZU;d$COBCh{wmBgE% zS5;|VK~{N}c++3Ldcmi`u{u`zY*uWH`JX7O(|K<`FfC^-G4ekxz+x*J&+RZ-An2%Y z>h1##YM-nlEf&0F;*D)tc=}n%x5wHM^{fH^#1{PFN%;Gv*z0%EF#Mg()xklmEI2EICulGx^0Lk7)RWNdZs66D(CwS3bMD#d1$|kf57OC| zA6U-1Ow2$uTiMw?^x`qyPZblnnf6MsUU1nPQ@Nzu$p69Zw-NspH<`$q*fB-avp3X5 zGMezcda19DkbBKV}&fcy$&rCh5<7bDO*xpmy z%y8h=&xt&%eg!=cpEO_4#MP`L)JU>{@8n`hQ-()7KA&!tV3PiNww%em#H#<-tM!S0 zO>3&F_We(``y7~clI?q1OxJ0(;}_HR$T8b@J=I@!x!7I!Zj8&#Um+8nSxm%@9u?&8 zS^6cT$gnw+;R3Vs*KLp78~Psa%<5V(zmDg0`GyOkNA(qh>p6ea@LU&ud5?e2WxemV zmo&HU?_%Ejn&G8#$%Wf5pFfx||L79_1k1+r>Kpc@sGpd#aQc3Z1A7k0p5zj&+x=qo zTk}K9xBl^%`ti$M?FEqmJuj3`Z819#x0q+|=f|rL$ZQqgQp0=ty|Vs)xeb-}yWTuM zrMqIYS;^*$yDuouKU6DG;(sjdmxNk9&+BqEg?x@P40|86xG?)olA^ z_~4_Q!{>9EzhBM0yn1?#sQ8)x_rF-r7T4T%+pWvRi*?&sJFac7p6|)g2k(IpRO3!XHG`K*v+vHP+7`&{P%l%?9@GO7vP_glIeb4em|I91y z^?g?G&tJBG-_qUB*D)=qXOpgHJnjGZwkAx;G{0C+zNe!=5quPnN^~V3Yo5K8KXGBIFQ06MxWI*V)l6pn47b*XC@t9E znaPs2tLw*|v|rM<;}5e}Etq=F@^b4G>5U=|)(@X8INB9q8f~s)E!I`)?tSafF=id* zJ2x*+zWO;p?L?}$@2VS-tWk@DZ*_jyg<{c_pT-3|Bi7hFD& zxjX*WKl7X=+KnZ%`rcJ9JALP;QrGmjn{}Lf+8IiwztlHaU_Fza!FSUxQOAxHacObR z1=_cFIIU;il6jcz!b_KX&jKa>+RcfoV+f8{bEub_t`+lDJ@|_`V&g6S`jbA01zKCy}D3`TYihJ+02VVBh+S4*yH!!@_%HVl1 zrI%sX!ti~1MrJPGS+`8f)G}OT<<>AWGqfoBGtYAw9gP+~)u^p&A82oz%lMba;J3;o z=DE}BTKTvNe;V9fUubD|LAuZH@#YlSe%IK%==jWp@Rhqd(v9a%3A9{sQbN#S`=T{# zw(VWTC_DKLbHlnB5oI4wsefy)G@n(tEylHfY5#^Qm#Q6Q&iq#|{kEOQo^@Vu!OHIo z#G2#IyKe6Yvac0nyv3M(#G{>Mi};GQ!QW1KC@+}Ql&-QS!}q^$I1^GHE)l3r&cJcUl6SoKT$TRC8}-P zv1PSkXN+gsE}s!5&8R7Dy-4_Z{7I`r)(@%#FSHdZbtc4zUZFaVIB98e~e( z=qQ`^MDmKKmrm%qtf=`MmBxB(47x|p@MKhd*4(?HZJw@AgWma+ukSPR4vH|kv2028 zX_+_WkMie3HrJHSc|5$6mwlr9W=^=ss-_)1vJ6|)CUd;V@i;qAZiT+hg6H2W&%9rL z>{W1dm8kIFr{TAj<*Z#UfA&V))rFODY}XG&KZ{h5W%M%*o3SEw)uP3>x_UVtO6qKy z&#ARh&d)>bS5D&=X2;@da({&Gewk4lDdup#zcGCMmRyEduUPFl;&YAf-n}2?;O&r~ zDtT}c>x*;8m}lk(nkCI(NQj;LiQ($8#YH>}XFg{;?7eW=M=b5dRfFB)SC&kivT*5q zrW22Xx|2N8ZhFst?z-vvrVN|d=f<`s4&6rm3|3E${#d|~5%qCXbh_G^2*IdDn?;+u zYc^+P*Ca-|9%b3P%G^KZ*6;XjeyJ?VcbGNB%OttiEKZ)SQ%AM;^f*v!VCA$u*UM&avO&I?RlIw$oOT-hCV zZ-VFPOPuZtZ=KTGw7h52{lf=~PU&8n&g-`Fx7sB^jR?o9Y7-nScS_8g>F#pv*de{g z4RQy%wHGgVlFju(M51kJu&Y38N$Bml&UL+vYeNmU?ut!tnElr8K##JsmcwpI_XT^@ z5@PLqcTQP!C#1;ak%YbXwneM8?S=KS=3jgxI@NK~&fC*ETfcN`rC4Yb^_Va{N_&&` zE86SyMWLxZcB|$;?NDv5xYzq``>#oJi;`SFMDR%OX*ia-xb5X8&cjCTi<`JhJ=)Fx zO+3Q6A$$2zhPSG=y1SeM=kso}IP%%2ej2~Fk7Wn%{TXJbuQFTZrZw=L+8@yU=(x4P1HLD05_(?)8^1AT zC#?G>TpE{_vpM$HJs&ByqKV?dJ8dm*I$iEr*pbWn(8fW1<~3CYqvE*>h0lmO>i%>$ zt`xbTJ7cxiA#Dy0w>Q!U?j3AbkriRDY{>g4Eh!(GAgEcdaWi0xrCHe?Wwz-IXP>RK z(Dt(RgosQibPL2@E&Be-tq6bc$N#h1Ygt!9nFO-3yro+ z+blU{A6xUT#rh5IvWJ%}IPyGcA!m!y^8AwOBZtK0XRIl1`)9rIzUDo{f~f^ddHFaL zEiWf^iT=D6CUSCS&}&sy2JOcsQSJ;2XRO+rBFEO{a-PBD{G*ud2Kxg_xSU*T|e$&4irnKPciZq?=Q)gal;^KV!k?+EtNbi*)Q}S9lKb7}i300M5lNES%srW-( z)aOJMi^G07Q_KsK-|+lyYBYJ|zgje;zWdUI%RXt0s#CNAnQwK?x_a}7H~&PwBf%G5 zF8t0q^=6UF6Y&F&Po}JzQ_eT<8kd(A|Ko;tDVJ}ltIuA&pzP1yH}5!4G{ilbz+vv< zrWJ5xk&^AUi%-QpXE!NC9oY59Av;m?M)r^IUNaO#-*+f14ejW;Yh2xxW_Br$!OMC* z3#0ChYiuRIuY9}|VCb;sh(G(vu=Z1C>lqhpopyTC>dn6y76iAM?a{gDXT~X0k@0-5 z_TqS!hOn!K46zS$O`N;WpVE?y(mEZq{ovFMOobvBa;N3Y&1dTgvYO*^@jMs9E~zH{ z(+9&AE@L#CRLb3Z;>Og5b??mcbrM1sD@L=icT4^(%UB1EYsPX=^8?F7fP9G z%{7QKT$COySl_6lmF68}eg3V7i-VQ0>3hxiA5~pz4OcGRu23(1Cx~6)W@e&t>I-hx z40GZ1+#f5&4YjLR*7rVKYPnuj8mNgLj@f7t?LcoIeK13=Wf%qg|f-+!s{~qRJ(U|E<0hyv0%lS zwXcpjADI2}J>!Kv;+$Ix`cxjhGfZ8*X=T{$3dwc3n}h?F2}~?)*e;f`n(^(__l-V_ zWmj*Bov)I;;{xj&%LKo#Rc@ymSu*Caa_1X=Udy?*A?S?5?3xc7t3NhPStEOVm5$^B zZ_(Aq+){X}9E_*V`pUFt?{k^vMY_-O4z9YpAS`rQP zCAXzoT0VKLcp%!Hx8&*szK$c`9t&_qyzRR0dGlK11y2_>_3O?r=lRU)4Dl-LD~Q{| zW0|Br$$gtpoPWyfmz&S-6j%Hfxvm zEMR#dxYXK*hv8Q5iw*^C#~CGO?yM)%f+%H?-=eH zrZS7 zjGcIwu`0Ge_6S?Yw_|SGbpvng-lP1~hbdX@3}f!Zs5nL$hAnQ#eYcdf-D56M^lOXj zY34urAys;aLjLv-gJCr6P^omgU7urW9`# z3*2GLd%NJwiakx+N}Fdf{N?Z2{nmzQ*2W{7POiDc#vH!I)FJhZP2gnjTlX$@3dJtdd#<2B9~o|(o!v@<7y~SWI-Uk^jdPUmdch{?yEKP-Y6@T{CgXk;*ECkh5AT+xoTzyUtDJLm@YFR0P~~W*v9a@^)dlw0@DQ#-2CgUcU}CPj%LGoUwdY#j`4o~Gt+kqAJU$ybn!jQ3>T;1 zy|USV<(4cd7yo0EA-XL0>${q3XT0a0vgcx|3e;G&apKx&N8S4o4Pw9T3Kzfs`|8*i zh6P7mj5B6`6N%uyA$CD_#jR`V%_)M(A#%qK%(nW(yu4NCd-YSb?wJ#R9NFIgj{Ak_ z+xE;gZj8JaUg`7lFjYz3(@;8<5u(G-AS7{n_D^>P-J5&0Z`QJ{KJp>OtniuTKF>Y=u1RdQ}>AG0F!1(XuZ|==3j#1hx94D8Ds2<-u>qX{ov9}4^H!Xg1V!{iQ7^Qh zzV+Vehv82oEZ88EhA1WaK7jE0{L#RS{d1JH_VYr69=^yO%}|A{Sy0Xf%A=BsSf0(f&wB7nh@4 zMcg45`TR7Wz7dz;wK~)-@Oj>{`cqMS-rF1AhR3c8sWiKJy6azu z{~n+9Gd<>*9}-9oz1I}Dc!9^`8wtKD=FQx~%C9`XY3+%c=W>3rWtToH|Ko;fljdb_ ziMYAaU9w>sPuj+grX@=ko@+L>W&c=wMr!WDa0inG8H{(sZfx-9agC9cZjNl&X2qbF zZDMeBTY-tQdlX-Ok9e==4Ymt%zJ>cAPrP)_{zfSTQmZYk5JK2_B;jqL94J@!j6HJWP|OU67G z@rQhj8IH|C-+GV4%KqVz%=VwTcT)D$=AO9 zeb)Z=HO;6{ChLk#jYg*fE^qAZI301VZpV)7Iigy%WQMV3spoVA?sGS{I% z_spqDCK^iollvH=I?mrZ?&EOMaN3_Zr%&J3Z@cLrz!t4*@9ZAc*KtU8L(jjB%qy;J zH*C_XH~)INAb#ojt`m`*851Xe*%7vK@5BTht7^}~x8qy7KV0#@@Lg+LL)eRav;Q{R zW#+x#d)xG6g^+OQ#$_tAD-vAXyzai5ZtbPZXFMTo^#tKc1uo$`KD#FEd}V*|LrmL| zb)I&K@;y>6-e1@DNX=4w$&}GQ?N=Nx`wX!K!v6Y`X2mJbQoOXKt<2)H)zSjhWmYdb z`nxM<%AHxe&1crUD}gOxw@v*{tXP&h!{a#P7L7{^a#Duom-lzfYfm%wJjW5Qzk#dQ z;DS`sy1C3wZj(9GMH#nf&t{L|@@VWh7X8X_b&SA55!TiE+n!-dqNdu?u zJAF)P7x+WExR?STb2V(crSg&|DC|6&N&ktkhXvR!R~E%A&iFDT zWNXao={eN`0%8kPb5>-fR!{qT*JACI;@{l4ZI^jxgndrBT(z=~El;uWe8aRT20s^} zHsNe<$E7ow)o$I@x|}+PchON?96OfDxhZ}l|MlZbfR&gK=U z_e^9ZPd3}MZPRoAWD0U#2x9UO3%~p^$0S@o+2PjI@7|SXGlCcxBm+Tf0df|t*I1_& zpde|XHM5{!#hCYk)6zGLZ>Mr>mMsYGTNfS9)Eu_SVSj-zgRG}k*s|P%fd`&+&1J7B z3wxSvlqQ+}_erE=GUG1pIP+dNogGCzA#-P&WiE{Azsq>5*k$(3Js%gZDrq>!ViGX* zt?YuS3pV{Rp0v=qAcgUaL;A6Tw_#e7FN$|+u`#@Oxs&DiiqCwu`vNDVXDpAslk@ND z&kI|vzUR)aX?SIl*nT{=h)Mmj=83YdA`7vXHIEevo;>?A;Xd;Lk(W;@r`g_3IMot5 zdz0)kelH`gS4OW6Z#(+<=OU&p$Lkea8s`@7xv=n?-^qD*92bf?eORG6`{)7Zne9y- z_vHBFdTrD-ZrWWcVmbPvo%ckD@43%c?ps}93Cg+GdGU6gv7)p3rimSk3u7~RHFjUf z+tguJy8fFo`|MTcGupoO$mQ@VFWBHfIatzx0r6sLi-JT}@eYX1KW67fV>=(HC zes^u0`EP5%H0RxG1B<%KDz9mE-|JPF_1Qa7<2d`Gjb33HEE%V#Os|OkxpkApuVjzb zbs65Q7Zf9zub$Tq(VFjY=~GkwP5B_TAK5HHZPOU;rf~+h38qfn7$w5}w*CBBfobV8 z7sQI&do=i&D6PDnH8EoMLGPz0_MXw?o8jcvzw-6vkj#~(5vx~wXv;7jbKtzN-AvM| zS8&hTKU1dZyk1afS|B;|R*8>_^pA-T8>R|!T%7bL$u)HG9_a>V$y?nAUAX?2Jl@mE ztkA1+QuK{kFv|rSVOIyWeHYblWbazKuhTp@^Kh1;K-3Y(1!pgu`(Vc*{ZnhEk)imB z)>jM*_8wAV-1WkW%joPmL5J!hu8+kM4&{e5kMXiL3q7rQ7w5-!A>JTp*4H$@!irmG zSb~2i=@#>*WSIZtW!iN?_|~q<@C)sM@jO5Ey|$Qp@n?ieo?N8NAgb5vA+&xF>=0NZpT! zB9;q5)=fW+7OV{})GmA(=CfIAdPfL@tkS;xwlIr zFLss;-xB`!6XR|$Z{c45f?=UkmD&5K%`>NSWJs<48NFq4-X6INavxOKGrx3C$W7T= zJMU?X_JXkFlJh2D6({AAw>E|*Pe|M#s~Z8z=rs|%_N z7arfA<+-8RPj1JWe``Z;H~EE~@|O4DTHwlVa8J>2PsXE5CO3`WN7>X|JO@hN_)i?u6^}xw`Y{Y|19Pgrjgka7kW1@GBaDA!HJ-SvFl2N=H&O~cv#pt|C$;^4Zh3Q}3Zt<6K+!MZcFO6ZizaVq^ z3{iRa#}1G8%y@rW=E^E%)wU#|)Exed^KA>K|FfL-@tunCJSBVM|9B|JuDfq@p$ipMNTrXx08{^|9dlxldG^ zx&*Ht(X}|uv?U}-+q&|cxmL%tl@;v@A%D;LCY5EZp2_@A>oSMv;zx=vM7Vx=nBHYS z>-v1rYG>_EW3CK0;|5;I3#VNeZKf(Jvt?(9EZy-=%5Cp7=Q-jmCE34iAElI6`6+EW zaiC1>mp0n%J<7>~NnmcT#8L~5#q_5h#ZME^^v|nEx9adjl$dDOw zPBGr#(Zw9u1H1394qTbaqw6rkf2;dq4J8x(vqpF4W;*6iSaQ{eXQ$4khZSiTQ;zRB z%otU*X*##t{BDIdhpGG4U7XBz?0(}de!&fF3sx89YEFD=&g`@@L#nGsD~vJ9*GKTk z)#Nb7S&tPj)s^MWP)KKeaf&l_L&G$tJL}F!wq;~?*WTZucy;!LwUcxZ+Mj&Tmb5|7F_2T{%~UlxJptem`q|X|T{C zr!!UCCRi_eyiQ{E)LNDvW^ZMQIQkEO~?l$L)6Ko7`f7bY}o5-~A z^d`Z-vlf(Tot9W2T(24(;$U*Q=!!yoTkv$!UZ5bVv zUsiayO8R>q@p$H^#5p-LSoM{{0`-&^=TuIM)m&(i&a~Wh;*7LXSM=Yd-|ozQo}B&P zu5SC6jlVuzV~t&U+g-!)rt`$^ALqA}GG`d__c8stA8jvn#3X*Mj>=ly?sJcw-?%Wd z>`4`wa3n_gA?MMTVac}V*QdOAxlnS`*#_54e;-A|@Xl$?<`39DckJ_?F|Rl6*=$Xw zjPnJ{7V|oEv!yh&@hywh6IpOQts*ou(Oq@-g4A36^|#6tgA8~6i4!;$rckH zS1FPm3g)Ln8q5+de=>UB9ucuP!x>q}6&5Vj*?qoo z)pGxAdHxIDyAAF{OuKb5s?SIHV+q6DV;aXqYv&y2+y5!-RklR-$}F#ydP{>JMx6cJ zzNw!5McILU4DmTy64!NpGCE{Cn>a*!p12*J!N+VRU9wh|+hA?8a;Tc~N50LT)HYb;R+TEIZ?LQh)ZD z1KFDcJgMRg&Tcd8sNZ*HvZ&JLtZYI;$pidf~Pc&h27vm>cANrLJ+)O!?>O7zPv7tw7^hm? zy2cw)Y3;=RCWup|@!nph=Ua+T_AhQqSs7wu9hqSnJ<(1?E7P~Z&U*5FImZo>PqGS8bM`ceL*El3*>V zdbF+K*B_o2=X2xFZR!8;HiKCthYM zxh0-q$yMJuU*gQX%X|LLOq9CT)B7UkiD>N^v#wt{f)4T=CsG?#_HV3JOMQB_gr{t` zNW=aX&l|f>8b!DW@?Y>P&|_k~wf^dl*jg=bfqQ$zk23pSkzjhURb>-TX~pjK4~!Ue z!{Qs`m&*M<)P9T6OI$LZwJiSNCoRU=Mqh+E&s{E&EStx$KtGl@!*gQBG}(`LHG=jk zWWDtCKU87=aLqJE?cM(x7-CPpN@=^G`f*LWY1re#c=zqKzVX@QvS$kx?-f5V@z%5F8|>U0{vF;~e=m;NJ+8es z{E>0$n^>!d6K}md%aZ$cURh<=2g5lP!K=QrZhEF~xLC3w?`1`Vc7tDC7@POh-6i z%zGA2@OJh(&t?B^b5=+Awa0AyCi{5}^;d*0Om-5Tuzj)hCgz5FzkU1#{xr4i+G_Q) z^W@@)cg$t-OYCMDeOaG;*mKU^3iVB2&i<%*oHpC?0!J)+HV!r5dtgY3sg=561&dD@=L z-vvqE7dd=bKfPzyDg%a$)z8I$>g~{~KL7ZQaq4q@=N{|EThBQz*z=dfPdrd#a<}O# z`^F2A{L&Ye<<7YLr{PX-!F~USDw|g9pH}EML9HyPm2->r=D$yP6n@S*edM{<0^Qqn z0S(s-W^apUaL`v_b@0C&oA8r;!`JB>>(Y$>E`NKL;mg$@pEx}}b93G>Ua<0Mnwm7{ z)r(hR{=8dyW#WsDp57NCy+Y0Lw@+t3Tln@?^prKPRtsYjCG3rFO< zm}kw7PU>4_$`k&wuKCX|^y&MG_Bv&|m{vyn=BMHt|Cq4NIn;3a!^~-aj*IxWO`m*z zPiPnO-GaHCChg@aHx8H`o@sc5M49ex~W~)mmF>7JA<;DV?CVtVZ%x4dd(E4Da7xUi$uYA^UOBAD;qq{&+{M z=wJ5vVtDd{mwXq*7{u!M7R+woVBYny^^Z<;vcP8X;$vZ|Tl!yDp3eBKC=qb&rdvf-XW*w^WPu%^?k#gzT|_;KR*7tqoXKEDd2zW zm9m`W{K?OIBle4C$K+`C9l7~mE2^cVmmz9F;LY=&6I18d8kNjmIfW%8U(Dw1PxnQ) z&vm|#ndIN4FYx||*)5*)$JfdPI_Q5kdhhY{5#zR#=8oq-&ttQbKlY1J<&k}3UHp-o zFRUAqzg+n8{pMTG1K;od_@emcPa)4TW4ZUofBF9KOU^hM&d+^7*TH?y4r${_0x1n2 z*B<1FouS8f&_;ZYsx!~Gxm`^{ee9a-3Fmg`9C=!H{A=^gw4yIh&o64~o47T(<7``= z;L`53E)9B%?lk+CIQ&Te@h3Fq$G=}Cui2Kreer((Bfj0s8SX89$tJ_MPny}jZS9q& z3$0>St?u1?F9O;cW%`n1)5|`y9=o=F@}zZ{U6XF=n>om{d@NMzu6)`2nn!n<%I!@X zU&VIES*SW()?HMx`^9#}`_m`P%|HHp{o=`W9B+>;W-wVNa!G!xtorinOXI4#%N}R^ zisboc$+J)Let+`8+@%a(4++0z53e{`_Oto>;VEG!xAJ7Z%=x+YLd>xknTE2<*)rXF zD=Qn=XQ^#=i&i!j)ZlVyUb;n6MeOc(oBNUO>`mX9=I`fs`KO<=zrM=s?u;)s486PG zUXo|1(tc1Q^uX43FJt+%pt64&YPAO2W%G}G4;EdZD*S2op~c5CUhZh%vr=8HS{1cM zgFVC2X{wg@lrJR$uP?67%j#Ota#L@TjMyZJImfhZ_!u(+!^85nU-q8x(2&D%`mXoK zo?AKxr`2&f>=*s+`nc$OA5ZE_bH>l<1zhj{e6iDcf86ENg_4D>Vpdz-dyoG)vvA77 zN;ZM--5*|=&no`Tu!VENnJbUye$$rw!f+!q>J#UE#qZk}J#}N48^O7)JH-3t%c&Ez z*^RdFy@))NyUX>9-h=mk36mLW`480apDVZN-TPd_`n2YwusaXxpMFUbsPJa`R`$!H zBf#G`QmQ{=;pt`R{N0Kicj7W-N=vRQ6};|c*fsO|NkxXsk82efVtXv3t}1y(YO@?Q z5tm-@+N{~3itk4it6CM){CxLQf0b+Yv2)b0nD^K*DNIjKn(StKCGV!(?0-fIdka-rN3&+XVD_D)uHq}w47<$Ta>M15SfWsBq%!CV%<6TX~hN`x>-uqGs*!JB%|K&1OtJ3d%SAz4G&p(p>YWezuPW)Hp{NK8tVp_TW z-q&l16PBO+>ewFg%~X9y!{b-e4AKr&IvB;zbp7JSsCy#Vta6h?@b1{KcLj`d*H2e{<3@UVc*Nm(ywM8wYN?UvakJF zV%rhhf7872VzuLky*vLdSRQ?^A(idxpBrI|)c3VrFLHbGolW&|*rk4#3vvzbzI0UB zHn>deX__{%??ke$-M>`ES%%NoNLVX{N-{111T4ivi`vo_uKV!HWxCgCp|Uf+$V7iHMb?eU9G zZ>Eji-&1XR^LEr1ZmzAWmU(We$DXsE;rI5Yo4yHNH-E8lIGIJ zPzFfd> zE|oima=j{_rdOco9BYG}nkf7fniI4t+}~>~FEi zk(?aP;-h!`=Z%O}-$GR{y^!%}%g6~yvREH}YisW3%;re`{LY`+AAZe$@cXWV z#8zqlWtWTeA1wNod*addi>dF$Y&OZVF`VW2Bb1j`#eObmKGTNt%EA8Y_i?{pQ)$go z`FV@wqHd$>Nej&l<{oik?=5j_@na5J-&@|0qvsg%OnTm=ZQM_Go6j#3dd4eyz+{W$ z+EY1~p04kjouKqOz27Z@*@>xS+UnQGv?{OfSUb`8!2eC2UP@^;dbV$Cwl(p6c#$cz zOT&qKtJvq6+*PsHBSXGV->{5Z$>Mm;JiSzVBU#ytyzci|O7{0|K7an)^v&i!&+ceU znb5vw(Q}uqPu;jX}4$8UmWgYLi=4mmrW$PIl#B!}xYcg4-2r%rLvraSk-?J?V$`z=qadX}v2sn8bQo2O&T=`1;iPf4>_ zc){23pJ5MfGhdkT>glH^`yaa8X1HvVeaJVK<(6xxjtP&eivP4N`fsyWPjQQ~D&DxI zL3-EH&3i6QTla3Rr{s*d^QCr-GtWue&y4Ey-L>;Y;+uMj2}{-e=FL85_xVrBGp|<@ zpZC;dTg{UAdT3um-Wucdx4}Q&n=Gi}b*YlxcX?^CRl}0a%3CfZ$6l$v%sP9gtmgBL z+zZN&yLLLV>S;T3TTIV0y6EvBR;Am?bOEzK^xtY#+uXUc7bGZHm2W;Jm@d6xx8Sn~ z<_ojW@?`KR1_&FZJ6yJY{$kl8-A$hYt+xNMxb|hq$>xar{mxFO&gm<(?+?h+o0OE7 da{E8~*+n-@k3T#j&cMLH;OXk;vd$@?2>?5u-FE;0 literal 0 HcmV?d00001 diff --git a/cmd/athome/static/default-avatar.png b/cmd/athome/static/default-avatar.png new file mode 100644 index 0000000000000000000000000000000000000000..c732d942b33e04f1be7a9dced9211ec676de1056 GIT binary patch literal 1624 zcmeAS@N?(olHy`uVBq!ia0y~yVDMpJVDRQ(Vqjnh{qXGw0|T49r;B4q#hkaZbA6_! zN*u3WVsTDN>99>ltFwt^AD@{)p|9<$64|ik3kkxx$-WWOiVti$8`c+@JM9qr4Bi_> zXQ%7x3Wx7}Skr9Tuj+lW%)yyaxK&gCkp-94uZDvb=WFME-Xq8Hx4!z_^Ih*=|F?f` z`F!X5*Sp>=Xl!IWW<5_jm#Oi9@x;fh=Uw{5eN0NfP4kO68=@s`NKTOPgXo@*x7^oZPi zyYSA@d7oEp^-SD6Y43zNDTj2}1dQEMwzseM3#^gy-~aDqTOz00`GeC^tCmVBY+BNm zd;GhZrN>@PD~`L7h`yAl+z@}K`c1{;Bh!zRXXk(Tcz(`~MdqijO|6-zv`Nf5@L! zJ>gUFE~dz)ouL~4jGop_eyX>Z>$-|Y5XY3%4Qh_zI{x-g?c&snl)lfoE?Qc|v2=Cl zjp^&siaOSH?BnRR_PPn%B2v7U-%B?w%UU4I{xEOx(#zEWI+w1l(H2VJ z9jwQvUZBy(}Rs;+JI>Omy;!)7f;oBlq|{?hw-z8`ftfyk}*8XLoikc0-_F+I+9nY`du_h@%8s|8Uu;C~ zysbR(O>0uuihOgOMN{@v*h_BhjsDKlZPno^>`~i#eNnqH^Zu;?Kh9<_*i$#Vf`6^w!w%I*89(wpz)f)$|I31&NSyrN-6;$=bW-eHH&C(6b zvbkKM^~@0@YbA60|L4xFk*sGO?8KSznW1QPu8ko;mG_j>&;E(d|Vi*WTYGzUk?TZxb}l z?rqzgwQf=0&g}A)Sr3+5l}d~2r|66H&kj;MQU2lI>oeD;Oe)&G@wd*>O`%$fCet_% z={&K#;uIPN3&^!b!L zqo>~Gb>Ai@igC^Tm+aCSxc6j2tHU`ZzuE&LtaqDR{dTH3yV^2qmz@@!t$T3J&k3v- z|E_cl^{LM)%v2`77wE~4TU5Zb_r{(OgBkAwu9&P;nz&Va z>efzWw`ZfU9!8& z&67FGkz3|X$jydTV)y@^J0}0)ZFNy*mG%pTJ+rFf=N|0a9`W+=enCD6F6FOB~ zh2=fpge*G5vHimzm4^2I7j*~TR-2SQ$a~yByQ$=xgw-FTmcqK8liZJgI?NJz{4?WW o#NpsVp>H!!d~90o^kjd*Yh$nGg-&}J7#J8lUHx3vIVCg!0O3p$mjD0& literal 0 HcmV?d00001 diff --git a/cmd/athome/static/favicon-16x16.png b/cmd/athome/static/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..4267562b808197d5b00b58f3ffdedde4b689b3c2 GIT binary patch literal 316 zcmeAS@N?(olHy`uVBq!ia0y~yU=RRd4mJh`2Kmqb6B!s7nLS+`LoEE4h8S`kauAr~ znXdTa@fud{jRGv(2OAO;mAJb6KQwFIxNmharl>PLL?mg_?z=PZ?wu)Mo4xSuwS-IU zk?ahmw*TLL4EZm4V)Zw_9g3=KWe!P`l`giYpY&fLwBnN=<4$G6Et{2g@tesDsPY}? zvWV-LSUfc(zG`o4!Cn=Eq&I60z5AV~vww+-8dHUA@YcC)-#Ri`6wFST7b~VQI?UBP z_U8I0Ys35%3tpOhvQ7G^f8Y~)!-A6Vf+P=Ug_CHvJE`*K V@_u*pU|?Wi@O1TaS?83{1OOS_e`^2$ literal 0 HcmV?d00001 diff --git a/cmd/athome/static/favicon-32x32.png b/cmd/athome/static/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..869f6df5fc6bdd2c78f2ac91679fb6814f999485 GIT binary patch literal 769 zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4mJh`hT^KKFANM!Cp=voLn7SQhVRV^aTNJ? z?x=|thh59mt-TMtJvLl+Di9Uyb#!tO5o~?npD(#vVPA`YK(@C0A+1{~XDZU(&E5Tc z-dtnj@|&8QZqBu=zyDKx$>z8EkComnIILR{!}_Ok-=YfHf4@FP`wQ;dE??*S|Hn`L zW!8r{`94nlr*iE;Zar)Ck25jrKOZjsbCcVzsfcR}cl^FDPj1dI{`XK~dBg+mziKR} z^&d*d-)ZY;H{*apnF2}VvG9j!;jSyANoi9X!T5Z+7f-?nuvEv zWYsV49q*g$Z@txZj1ErwIxn&Qfm6z@%&FX4JQE}hjz=UWx$P?9;E!zdoojl>rNTrb zKxvv*)Wv+ujS|d^i;u5qXl;Bd6+7culEw>8f2r7j=Iv$qJ7#WB7P%l6UFx`dNyOil zl(YIJ9B;H(QbgCVE*IJ3SJkF+aPuT?k?U%TuL6&nw%9s1dPFoFe#N`59|wVvq6EYR*NG<|*i0k8ARYPr*$$BWLNdh$nm~)O+W164QsIPo3e$=4;Xn!Z!DEi1>8;y2_Ji%KcnffBD=9 zzh>(wfA4KQU+wdp)8BkbHmhp9yJ5~U;Ry|ozwUW`z4+1BJ=0{@dL3J2zvlTlS;o7H e^S-_QS+9J7zr7+tu!n(xfx*+&&t;ucLK6T@oM(;z literal 0 HcmV?d00001 diff --git a/cmd/athome/static/favicon.ico b/cmd/athome/static/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..b4290a797a9f7ae4d14373ff1608be81d5940725 GIT binary patch literal 15406 zcmZQzU}Rus5D);-3Je)63=C!r3=9ei5Wa>W1H(KP1_lEI2tPxOf#H}a1A_(w1A_oa z9Roz10SEZs%fR`!kD>p6AH%!9eGLEp^)dX%he2wFu5Xz|Nn5|LOgOXJ^2j(|DoeNG%<8Ovicl`|NpWX{{I8fAPi!IFo+GpAT|ht z*dPpIgD{Al!|?xa2E)I984Um7I1|c-(J(m}A4Eg(-%N&o|5F*>{Yzu`|1Xu{KbQuw z@nIx&Aie*S82bOFF#P+M!tfsjCnNDeY!HU2NoM%}Kbhg*-(-gV|Dg<=f0G#c|0glL z`%9%T0aUa%eeL3V=e?qv}A-@~x~e=oz|f4vO<>4E?EG5r1C3$|0}e>X$n z{~m^4q?FOrQ2W1|;TPEME`}5TyBYrd>xP!|AR2~2Wjl-wqCtER#>WQ9VZ$JONcevj z!@vI^yE_=Zfa@7(-GRaesYezE@j)0Q2BSf2QZZET|4xQ4e>)id{p(=(j~9dLC_-vr z@*uas*vK@@3>e+P@E>e9w9fm7PJ`5;i=p$;)u8jy)nntgGW`FCjzM)Qx)_X)E(hX+ zFiakq28qKkOdP~VX2a@ikT{G@E4qo{|37rIL3KS0gXFQXk;Sm7K^6n41@U1R#7D*; zaS$IwgK#~x?SV|g*!bEnNNWD$Q->^vqzBYC0_&?|`2TMJ@PC+{HPChzh{lIOZ83cE zAUR^xfYgC7RPFyNhX4OSxEe~MvHzosnUgW{|_53WBC8S9D;GlgWIQQdO>Vc^T z(I7pbwm1xf%!9E(G%_E=hGCE#41>f#Y!Ho(LE^|5BoE`m*aZy#{}F>hZF_v`2+8Bp z3(^PbM_|LaS1EO;o{{I7EP`?92W5Xah7)F=JB@R;uk_WM2G>DCi zL46)-;VfvMixl%feI}4T5RDCk(*x?)!q_kxBnHATK8yy5A!Cp@ zG9Q_p%JBbx3d65IAiF~ug#M>6?Ejy_@E64m$m&6TW^_I_F_0Q`agZ1YL;B{R{y7N$ z1>2bn>4*N0Wl;N{#IXNw62mW0|NS4x3>XIW(Lrn&4Pujt|0gm0`wy}siD5s;PEh|7 zhW~@YArTxGC;lffd;yIIfX549_QGhG-5@@Z7-Sa6%>PLYC;o%%2!;3?X7@;@LE|I_ z3=9kjM2?ey!s`Dj2Il`A3^M=w8G`;#V3_;ApW!NK%;f(BhQHJpFCoQ@|NRVq|4v}| z_P?Lu=Kp?%1^*{7g#K@5Q21ZZzyfkBI{shJ!12GA!Tx_A!~Fk!43Gc!F?{{o$M73u z&VSgL4`@6EG_HbS97My!kZ91@&|i=r`oV7a`oEXq+5diq<^TH_ zT>cj_a3Z@O?Djr}u>XAw=l}LGeE-|a@DDVm0~_C=hz7Y2gpu4vs(B!{{0F(Im*L0% zK88#G`xv7C*E4W|{O~`Nf%$(ogWdlghSUE+;oZydkLa-`&=@|Q#->1Sg8JbP*bn^- zo}je!zl%ZQe-Fc)|J@AV{^OfhfVmUoW)KGPK{N=1*vJ?ph7A*=23a2#^&mI??P2)w zzn5Xv{|*MZ|J@AU|GOC;{{@Y8A?u|U8#LxcEwiBJfZY1Oo8iU(UWU;B-3+t-_b_~c zkJW+f1C3*WXk-i$17Tz_5E~>0!yrBg!}uT?oevWOi4(#gwOBB8jPKk3E`|mFI~lJ2 z?_&6cGM@w*E5o7>MGZCNiLnQy28NOB{NKg!`+pb1^}ihqU;lS9{6!g`gsDfTLHeqC;FHrXiO1Vj8rzrKZNuk+W}&OFjNm{j1=T{ z@c1l9oKy@N^TehHmpHmws5{{EnCSB8e2}>yj7uCOhYN%B;F7~7j!zFhIb7!95{Hf1 z;-W$0x47hx#c`>FiNk1QJ?LzZ8W_eUhb#_LODPQ+JEqhOkbaOk$QUF>2&1b<=7YwO zK^R#KosBLJ;)5`{I!gF3^FTBRBbx;p^G3!Xc@QQf29pP=MaJmj$b1+ZnMT$NV#DM> zY!F7~BeOwrAPgHv2gzY$gTz2`APi!IFo=y0gVca9lA8ZW?0=-NLFT|PHhrM6e{>8| zk1Ph_gD^-8M5AMnI4%rT3!YPevO#kU7#OA>qz0J{M_)T z=H!s{VazAO^gw&y{LFXgu$0vuZ z4#ozl2Voc=L?h2R`Rs&;$)S_dU95M}3gUlzyMph4EgD^flAbA)D&2PilglLdj zWDHXe5`(c}G)N4Dk@+w-HX2zi7Wx0kVj#7+Fl>GtBoAVPXi72291un~2P6i<=;DO< zpm}$YJ`e`631N^rbPUpiE(YR*#BpJmJd7rCUBN$)dSr~te2_S3-2jLNVPrOlPb>!M zgJA-C!0RFisQHH`|38D_?|<|)@i6;Hp<(8dqK5*t&@~o+p=%xfg4bEV)<7VwM?evW z$-`)nIvAT;G_skx;akUR{7_-PFP|E4nh{-4Hh{eLRM^#7?0U*Ky^K2^%JA)f3d7w0sSKX~Qy3m#-ID=Y-vKiZ#0Fs)ADsrt!7#cUj1QAXr_tp>d~`K1 zek#L%kX!$!Fg*L8#t`&BoVGVQ+v>G%I+hV%au8C*bf@8CIj4+i%Ci41}NlNnC@O=9>8UKf+h@DIM;2eh6B zY8H6y4Kf>94kk`bI*H*wbbSzLjnMc1$qc9eCo+Vg?*;mA&%p6NmcjmiBE$UuNeqww zL;UgkZxX{_kbl7L2Duf4VeSL5K^Vq|(I7Eoj7uC@4H_HjW{};#|0gnh{h!3}?0+J| za%i}ttlt2|0SJTBQVfIG|3n7Q|H%x~|0gnBhOSTgOOgB0+=gNn4!Qq{41b~SzWzUv aVb=d72Jin73{tQ(19I;u9t{CXLjVA0l*gw4 literal 0 HcmV?d00001 diff --git a/cmd/athome/static/favicon.png b/cmd/athome/static/favicon.png new file mode 100644 index 0000000000000000000000000000000000000000..61cf7c943b6ac083ab0752f283391e37b39c1221 GIT binary patch literal 6405 zcmeAS@N?(olHy`uVBq!ia0y~yU~m9o4mJh`hEk44ofy`glX=O&z;G-z z!ZXd+mqCkxfq{d8jZuh!8Ki`Pk%56xih&i(W?%?ol!mk87&V}3m>3us4H=ji7#Ktu z7#PypGZ|Q*>T4Jvzyrz$(*cnkaMm6T-LDnNc-DA*LGq*(>IxIwiS zrKH&^Wt5Z@Sn2DRmzV368|&p4rRy77T3YHG80i}s=@zA==@wV!l_XZ^<`pYL41t;B zl3JWxlvz-cnV+WsGB+_PzqG_wNeN_;0t`UhnOc#Fuqm$?>aFDbTz!y?zM-ChJ~nL? zxdm|dq3EheEWoL$Br^?%T_72VHgI4-oM;1bffYEyBV96cQ}c>5^YiS?Ol-hT!;nCD z4oPDKE{#YMNE$103#^=rQbEZpBtJjL4s1DEYO~QtSB1@hRF}k(R69@#(={~EH8co; zr7&FsQ!4`lBtsx7(ai8oElEsCEJ;K+#n8aY#Kg+T7}*pQwMgcG0@W%aD7830rxcVF z?97b}3=9nP4MC)hK87^9R)o&byv!0iBd8WMVRT)Q2zhK4B1@xd^)Jdy%_~U+yVVeC z8?qQwy^TI7O(UgoaGHUL2D!M|aoOmDvl=Md*>N2@-TRY)K}5#W#WAFUv3B;}zMOEu zR{!s^b1e60dMx=C{DX0V#{_2gKdlX{lja{3Vl?dNd2p&zRqlZQLEniAr7zx`^4YvE zKXmRp#xrTZ-o%Ek4*UD2)^EObn%?pM()PRmKG-2AFaO`l?$n=yPs^%nt@YkNEZQk^ z{B4=^`_qCScRVes?0eUh>$5QdjxzTrK@_b-=Y?-@@Q7J=^qh{Fti|jNbF7LqZKoLB__=1f`!zGO*&l0a?eOb<%lR>EDIE6K`)gs`^Ov*FCkwy9)DWxlG&o=STAKuTJSFuJv3^x%+5h z*_`L5HLsrb|9@Ly9}_RPvT);>^84{A&-uIZIsVLw5m%3ldD3&J_jJ#)*&3%ZynEBk zFIk>5?>2B+^Z2qsjCGi7#PrUGNxTOxt*J#Kg6`HJFr?>VJ@M@6K0BPAM@eS)UG+%9)Y*x<6{OcwU){ks zQps888_IE?Bvwafy<8-8?+VMl4f6N+GT~((UDj-BXPuPTp8BSvWP-R_^N)5lj1> zvh(cO%PoJbsxJLw>$v{j?2<#A&lUwS8Ahf~x&GU- zP9;|(-@t~9m@%5`RrJO8{}C+Z`!+gh{j*yNQb3m&?1Tz_#!_RfrrTNOhS%$IML`jX}N zYRA3|S%)_oK`oK9r`~mxeQ)8iN6!EFaevK!${Tp?I2j$DZ&vV=GM>p`rkpA`TU~9l zr{U~*caQa)zvmV4z2xVg2S1q(<#S8&uueU%peCx3ByVyubd!>!;1j8RB2_2e_{>-o(?t zyE31{Q&{Emo^M@Bj+O;88qd6Q+Y*po^6|j5H&wGzLNfVwNL%bX;`q#I(Xq&y1XcaS zIZu|JkeF4xvcuna$y4cdbF0sCR`~PjP3Y#hBvIJ4&Rs`skz``Y&4Y6J2Yy`3=gWQa zYRRchQ;wF&F8R1;ioCdd2fuZX-u%a*wYizGI$eKsCbS!!eafooEx+x-iqq!{4zRZ}g4@TB|-Tt1YaG?%-Xto8{@@iZ8u+{x?<6CjZb;Fx$2&rl9D_ zhba%#t)?$HpghTT{#nThvjco;A34TOaBqzNtlt*xpq_pIu~x($>o=SRNj&MxBHIO4 zoHeotVK|t0;)ITxcjcD!-XI@FJ9!PU`Nt3DopN~m+C-#W#J9oV^Ry7Oi<-I#h`p@%8 zO*g&%Svz1|X4Ira^^ASNxX-S9b-%T?4T0oZaLwi^`w5Az?{W4 zoSE0Au3z-kOo7S4W@kz{&kW&*v*iz|ai+E97qLXS_ucr#zWL;~z#GdvBX@C5EX>f} z(@gi_45rXubEHwI7!FJDhn?un|kYw zmXhl)MUIn45_fhuKU(qnmT47RUD1{0qN|!x{p1{sHRUq%R_jW!#Qdl@dFW5{5tCHm z2!Ud@oky>)cjS6-|8Cpi4e||pUi*i=vvK1}IP$>R&qm$h!QP016^qgjF^H(DpE-8a zVe#(`hg~jTO}OxkNw4UOMTq}gqw~6~=7sNyA4iG%9EoW;aEIyO>xZ?iLWL4Gb}W5O zPNvc!GpfoS%S~H!O>M1s>E0{quf_8OHt$~Do;2(F3e}~Cd;exk_z-h)tz^MhU4geM z-iPl@f3U6M<75Zv4Y7ut9nt|s%Dx7Xizn1fp7Wx7`iTYve+%&a#BdMtmt)BSlq%+>cd`EVum;W3e9l^rMY zE1#zQ*W~$oMsOaB+qNS$&wp>7TF&guvS+`rg?X{qtxn&9&CbH?&8e1)K5>@KKkd7j z`SGUneAo2nu`(QruWEh7+SYwxP3D(llZ#p2243i4OPZMaq43!MuEpQ&|1{JXb_wn5 zD83szU5dAjx3%DWaDzhoOkm1~y8?RpicCnI5X zXzI~Cj;yXy_et3)Nt|kKBD*`UZQV2P!!zeg;gjs-D}ou0dmr-E33_~|d(({@ns*LZ zJeYrT-{qEn5gMFpWBJQ@?h3g`+a>HxX_8>Iso9_^7`pR^HS?dy#U}B$uguvnGq~i6 z2+xMU8&2suFimh`S+q&x(NF)YFYYEiOKg}SJ~?yNvyDHKSb0{RQ9C&2V)^$-e%Xy; zx0l-=w~#!@d+<}A&U1~kHdU3$J3gMTaA)x|c<@>5lvU%CqhkB_u%CICcl5`E-cKdP z9_OB9mY0b1uXK#)37^6nOVa<*`?xwcwCK~a&4LbJfjM%@NU)5p92 zi)r`?b;t;)&Q$YtSj{P+KF=LIHoJNU+h>N7f&$AO+GLF4c%M;D+;J;_K zWSY>KcP0;4tY~hjU`X5BdGMxs&74kGt|R@?c-CiXm>$#0+;JKVb z4J-O*GR>LYvS-CMhDRSd&$Y*>6@3#+eEvuNSdR+J+N0@?b$_((&iLRcwC-2)nm4&q z(p%nhBvkA@Z?gBLPw~NalmCGx4EyyPc?-|7HmgV)C2V;6^W*nlJ8~}G+F{Ui_sJpO z54lWwYRqR`HY&RGpI4TzG`vvxCG{QaIZw|+m)EN=KR@?t{z3Kc7D-rJU!PCl9HDsmb}bRz&h=7;#w`c7akJHZI-+GbPdnN<*uB>w!f&xoMq>FZEmx| zIz^^pdpnO9^#g844UgPT&3@k7-LBNAVOG{}nRjq;8k;whdcM!wh$%lAJtoI0t(q61 z@zH($#oGdVoJymwtVrnc&W#N1J0`RE*mVD*^vln-A6k6oNQOh%exvKWeOoyf91aTT z_APqTRgt6HclE$IS>uz|0UQ#c5_fMEc3*ltuc23#)py&K7YDQ#-&*we^|jMRPHeP1@MhBz zU!z^Uh5UjB{O+=w;`$}Fr!dUe06$F|sGjVeap zQ&LXOVqh@rT=~Y_H)a)&{EnIfKQb2>``>?hJU+)^+qAn!`D;pk7+Lk}mOU=t^kTEa zh8u~>XMfmlwC0n`STv=MUQ8X>@g32~Sv-&p|pDJ41v%Bc{t@&5op3RRx?tASde=wgde|Ey-w)~&l z7o2$|eR#4!t1r{JS==!XjC>}!?K~l49ag!=W%_r%+U1k?otUatS9mk&Xg;6dgD+oF z&u(`(u#fAsGaE}<%iJU9mUt*KbuKPeH@d9wRjW`riKXs@fL+BVlU|*!j=rsBbF6Ri z#5P^#TqLnO@A0N*PqQE0G4EXPWQNDCkTyv!bE!r~1+SX(^*k?KmPs))IV40FXYHST zF;Vbfldwzq;tP{k)LnODYPq+7Nr`WoraXIVDDMuxAG_El_C)Zrwd@K_IkxA$O!~!# z2JJT;YAtqNw6BcuXm?;Zeg0C*(`^%2`Y*}}PH%J&ySTgB;$nBT&H&Tx%znaci zq2>M2=X7!(OIGA_-79BT)aTwUtaKG#YxyiIFZ2o5{zJC&cn`X0JUHa=(X?Xs9%J@^ zRYE(sXZ}j)KDOwU4yTUrssl!sMScdYQD*90n&Dh@M)F4Eg^3%dx?k+JVKh}Ph&2tB z68XPS)8EA`PSb?wmHZ8t>#5@J+CC0EqL>(B3dA!%TVIz zCwu0NN}Vwr>yF6Cw!Qe{mbh+P_YEDt^pGP@tTrC>Ka;yI=5hMzrisk+Up#!yyzc9+ ze^&ox_U%#q=qvQ-uN=K~W?y#PD}G&dpS`NvwD;csNaI80ea9+iZ}@rHXvV4;Ok!o%nGIK{P4YHO zerKFgrtxj5vCJNQw=F*&%I{TgUFN=l^D@sqr`#E5yK*PB-)^+ycClL%uE@5>?bE3b zw>7m{MI_IkkN>FjMJ?p~?&^|_bN#b1^0s#WCwadXOpP1BNVH&pL07k`*kamYyA z+B5Bz@P_Td`PQ~l-t8wo95~w=(R@=b2z%Xu*dPekgol=(l`O%;n@m%r}#`%mtz z(@%?kShM)QL8rpcqMLzCjB8rma^o`gPObVJ{(nLUH=F6v!wc_bF5<2CWfd;3xg)Tj zuX{1)k(3`B&fWTw<1i(CGeg77;C8!uZ;!V+Kl}daG~~w=GxBiWb<$hqwJXgfL zL7TgIUF%X$6V??wi3>tv-Nd*;II z-KX>W-Y98&ko!!m6O~vf*GHm|G=8a zopslL)c#v$P#-3KsDILRcFw<67eA?+v3LZ^gcg~_aZl?yvT4OGi@zWb zs_?i~zvWSPF>hgsf%NylZ%B3eZzmef?KET++p_L Q6Eu|X>FVdQ&MBb@0K66k1ONa4 literal 0 HcmV?d00001 diff --git a/cmd/athome/static/robots.txt b/cmd/athome/static/robots.txt new file mode 100644 index 00000000..4f8510d1 --- /dev/null +++ b/cmd/athome/static/robots.txt @@ -0,0 +1,9 @@ +# Hello Friends! +# If you are considering bulk or automated crawling, you may want to look in +# to our protocol (API), including a firehose of updates. See: https://atproto.com/ + +# By default, may crawl anything on this domain. HTTP 429 ("backoff") status +# codes are used for rate-limiting. Up to a handful concurrent requests should +# be ok. +User-Agent: * +Allow: / diff --git a/cmd/athome/templates/base.html b/cmd/athome/templates/base.html new file mode 100644 index 00000000..a452d128 --- /dev/null +++ b/cmd/athome/templates/base.html @@ -0,0 +1,55 @@ + + + + + + + + {%- block head_title -%}Bluesky{%- endblock -%} + + + + + + + + + {% block html_head_extra -%}{%- endblock %} + + + + +{%- block body_all %} +
+
+
+
+

Bluesky

+ What's Hot + Hell Thread + Prod bnewbold + Staging bnewbold + Staging paul +
+
+
+ {% block main_content %}blank page{% endblock %} +
+
+
+{% endblock -%} + + diff --git a/cmd/athome/templates/error.html b/cmd/athome/templates/error.html new file mode 100644 index 00000000..4944f5d4 --- /dev/null +++ b/cmd/athome/templates/error.html @@ -0,0 +1,12 @@ +{% extends "base.html" %} + +{% block head_title %}Error {{ statusCode }} - Bluesky{% endblock %} + +{% block main_content %} +
+
+

{{ statusCode }}

+

Error!

+

Sorry about that! Our Status Page might have more context. +

+{% endblock %} diff --git a/cmd/athome/templates/feed_macros.html b/cmd/athome/templates/feed_macros.html new file mode 100644 index 00000000..3b990077 --- /dev/null +++ b/cmd/athome/templates/feed_macros.html @@ -0,0 +1,106 @@ + +{% macro feed_post(feedItem) export %} +
+
+ {% if feedItem.Post.Author.Avatar %} + + {% else %} + + {% endif %} +
+
+ {% if feedItem.Reason %} + {{ feedItem.Reason.FeedDefs_ReasonRepost }} + {% endif %} +
+ + {% if feedItem.Post.Author.DisplayName %} + {{ feedItem.Post.Author.DisplayName }} + {% endif %} + @{{ feedItem.Post.Author.Handle }} + + + +
+ {# TODO: relative time#} + {# TODO: parse and fix link (custom filter?) #} + {{ feedItem.Post.IndexedAt }} +
+
+
+ {{ feedItem.Post.Record.Val.Text }} + {% if feedItem.Post.Embed and feedItem.Post.Embed.EmbedImages_View %} +
+ {% for image in feedItem.Post.Embed.EmbedImages_View.Images %} +
+
+ + {{ image.Alt }} + +
+
+ {% endfor %} +
+ {% endif %} +
+
+ + + +
+
+
+{% endmacro %} + +{% macro primary_post(feedItem) export %} +
+
+ {% if feedItem.Post.Author.Avatar %} + + {% else %} + + {% endif %} +
+
+ {% if feedItem.Reason %} + {{ feedItem.Reason.FeedDefs_ReasonRepost }} + {% endif %} +
+ + {% if feedItem.Post.Author.DisplayName %} + {{ feedItem.Post.Author.DisplayName }} + {% endif %} + @{{ feedItem.Post.Author.Handle }} + + + +
+ {# TODO: relative time#} + {# TODO: parse and fix link (custom filter?) #} + {{ feedItem.Post.IndexedAt }} +
+
+
+ {{ feedItem.Post.Record.Val.Text }} + {% if feedItem.Post.Embed and feedItem.Post.Embed.EmbedImages_View %} +
+ {% for image in feedItem.Post.Embed.EmbedImages_View.Images %} +
+
+ + {{ image.Alt }} + +
+
+ {% endfor %} +
+ {% endif %} +
+
+ + + +
+
+
+{% endmacro %} diff --git a/cmd/athome/templates/home.html b/cmd/athome/templates/home.html new file mode 100644 index 00000000..9240859c --- /dev/null +++ b/cmd/athome/templates/home.html @@ -0,0 +1,20 @@ +{% extends "base.html" %} + +{% block head_title %}Bluesky{% endblock %} + +{% block html_head_extra -%} + + + + + + + +{%- endblock %} + +{% block main_content %} +
+

See what’s next

+

Bluesky Social

+
+{% endblock %} diff --git a/cmd/athome/templates/post.html b/cmd/athome/templates/post.html new file mode 100644 index 00000000..694e608b --- /dev/null +++ b/cmd/athome/templates/post.html @@ -0,0 +1,46 @@ +{% extends "base.html" %} + +{% block head_title %} +{%- if postView -%} + @{{ postView.Author.Handle }} on Bluesky +{%- else -%} + Bluesky +{%- endif -%} +{% endblock %} + +{% block html_head_extra -%} +{%- if postView -%} + + + {%- if requestURI %} + + {% endif -%} + {%- if postView.Author.DisplayName %} + + {% else %} + + {% endif -%} + {%- if postView.Record.Val.Text %} + + + {% endif -%} + {%- if imgThumbUrl %} + + + {%- elif postView.Author.Avatar %} + {# Don't use avatar image in cards; usually looks bad #} + + {% endif %} + + + +{% endif -%} +{%- endblock %} + +{% block main_content %} +

Post

+

{{ postView.Author.DisplayName }}

+

{{ postView.Author.Handle }}

+

{{ postView.Author.Did }}

+

{{ postView.Record.Val.Text }}

+{%- endblock %} diff --git a/cmd/athome/templates/profile.html b/cmd/athome/templates/profile.html new file mode 100644 index 00000000..637c190f --- /dev/null +++ b/cmd/athome/templates/profile.html @@ -0,0 +1,67 @@ +{% extends "base.html" %} + +{% block head_title %} +{%- if profileView -%} + @{{ profileView.Handle }} on Bluesky +{%- else -%} + Bluesky +{%- endif -%} +{% endblock %} + +{% block html_head_extra -%} +{%- if profileView -%} + + + {%- if requestURI %} + + {% endif -%} + {%- if profileView.DisplayName %} + + {% else %} + + {% endif -%} + {%- if profileView.Description %} + + + {% endif -%} + {%- if profileView.Banner %} + + + {%- elif profileView.Avatar -%} + {# Don't use avatar image in cards; usually looks bad #} + + {% endif %} + + + +{% endif -%} +{%- endblock %} + +{% block main_content %} + {% import "feed_macros.html" feed_post %} + {% if profileView.Banner %} + +
+ {% endif %} + {% if profileView.DisplayName %} +

{{ profileView.DisplayName }}

+ {% else %} +

{{ profileView.Handle}}

+ {% endif %} +

@{{ profileView.Handle }}

+

{{ profileView.Did }}

+

+ {{ profileView.FollowersCount }} followers | + {{ profileView.FollowsCount }} following | + {{ profileView.PostsCount }} posts +

+

{{ profileView.Description }}

+ +
+
+ {% for feedItem in authorFeed %} + {{ feed_post(feedItem) }} +
+ {% endfor %} +
+{%- endblock %} -- 2.51.2 From fcf9bf203d1a572bf591d03f5140e31da0fd520f Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 19:14:25 -0700 Subject: [PATCH 123/192] athome: refactor to new house style --- cmd/athome/handlers.go | 85 ++++++++++++++++ cmd/athome/main.go | 63 ++++++------ cmd/athome/server.go | 218 ----------------------------------------- cmd/athome/service.go | 202 ++++++++++++++++++++++++++++++++++++++ go.mod | 2 + go.sum | 4 + 6 files changed, 325 insertions(+), 249 deletions(-) create mode 100644 cmd/athome/handlers.go delete mode 100644 cmd/athome/server.go create mode 100644 cmd/athome/service.go diff --git a/cmd/athome/handlers.go b/cmd/athome/handlers.go new file mode 100644 index 00000000..e81cd119 --- /dev/null +++ b/cmd/athome/handlers.go @@ -0,0 +1,85 @@ +package main + +import ( + "fmt" + "net/http" + + appbsky "github.com/bluesky-social/indigo/api/bsky" + + "github.com/flosch/pongo2/v6" + "github.com/labstack/echo/v4" +) + +func (srv *Server) WebHome(c echo.Context) error { + data := pongo2.Context{} + return c.Render(http.StatusOK, "home.html", data) +} + +func (srv *Server) WebPost(c echo.Context) error { + data := pongo2.Context{} + handle := c.Param("handle") + rkey := c.Param("rkey") + // sanity check argument + if len(handle) > 4 && len(handle) < 128 && len(rkey) > 0 { + ctx := c.Request().Context() + // requires two fetches: first fetch profile (!) + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) + if err != nil { + slog.Warn("failed to fetch handle", "handle", handle, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) + } else { + did := pv.Did + data["did"] = did + + // then fetch the post thread (with extra context) + uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) + // TODO: more of thread? + tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, 1, uri) + if err != nil { + slog.Warn("failed to fetch post", "uri", uri, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "post not found: %s", handle) + } else { + req := c.Request() + postView := tpv.Thread.FeedDefs_ThreadViewPost.Post + data["postView"] = postView + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + if postView.Embed != nil && postView.Embed.EmbedImages_View != nil { + data["imgThumbUrl"] = postView.Embed.EmbedImages_View.Images[0].Thumb + } + } + } + + } + return c.Render(http.StatusOK, "post.html", data) +} + +func (srv *Server) WebProfile(c echo.Context) error { + data := pongo2.Context{} + handle := c.Param("handle") + // sanity check argument + if len(handle) > 4 && len(handle) < 128 { + ctx := c.Request().Context() + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) + if err != nil { + slog.Warn("failed to fetch handle", "handle", handle, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) + } else { + req := c.Request() + data["profileView"] = pv + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + } + af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle, "", "", 100) + if err != nil { + slog.Warn("failed to fetch author feed", "handle", handle, "err", err) + // TODO: show some error? + } else { + data["authorFeed"] = af.Feed + //slog.Warn("author feed", "feed", af.Feed) + } + } + + return c.Render(http.StatusOK, "profile.html", data) +} diff --git a/cmd/athome/main.go b/cmd/athome/main.go index 084aa70a..cd0619ba 100644 --- a/cmd/athome/main.go +++ b/cmd/athome/main.go @@ -1,30 +1,34 @@ package main import ( + slogging "log/slog" "os" + "fmt" - _ "github.com/joho/godotenv/autoload" - - logging "github.com/ipfs/go-log" + "github.com/carlmjohnson/versioninfo" "github.com/urfave/cli/v2" + + _ "github.com/joho/godotenv/autoload" ) -var log = logging.Logger("pubweb") -func init() { - logging.SetAllLoggers(logging.LevelDebug) - //logging.SetAllLoggers(logging.LevelWarn) -} +var ( + slog = slogging.New(slogging.NewJSONHandler(os.Stdout, nil)) + version = versioninfo.Short() +) func main() { - run(os.Args) + if err := run(os.Args); err != nil { + slog.Error("fatal", "err", err) + os.Exit(-1) + } } -func run(args []string) { +func run(args []string) error { app := cli.App{ - Name: "pubweb", - Usage: "public web interface to bluesky content", + Name: "athome", + Usage: "public web interface to bluesky account content", } app.Commands = []*cli.Command{ @@ -34,29 +38,17 @@ func run(args []string) { Action: serve, Flags: []cli.Flag{ &cli.StringFlag{ - Name: "pds-host", - Usage: "method, hostname, and port of PDS instance", - Value: "http://localhost:4849", - EnvVars: []string{"ATP_PDS_HOST"}, + Name: "appview-host", + Usage: "method, hostname, and port of AppView instance", + Value: "https://api.bsky.app", + EnvVars: []string{"ATP_APPVIEW_HOST"}, }, &cli.StringFlag{ - Name: "handle", - Usage: "for PDS login", - Required: true, - EnvVars: []string{"ATP_AUTH_HANDLE"}, - }, - &cli.StringFlag{ - Name: "password", - Usage: "for PDS login", - Required: true, - EnvVars: []string{"ATP_AUTH_PASSWORD"}, - }, - &cli.StringFlag{ - Name: "http-address", + Name: "bind", Usage: "Specify the local IP/port to bind to", Required: false, Value: ":8200", - EnvVars: []string{"PUBWEB_BIND"}, + EnvVars: []string{"ATHOME_BIND"}, }, &cli.BoolFlag{ Name: "debug", @@ -67,6 +59,15 @@ func run(args []string) { }, }, }, + &cli.Command{ + Name: "version", + Usage: "print version", + Action: func(cctx *cli.Context) error { + fmt.Println(version) + return nil + }, + }, } - app.RunAndExitOnError() + + return app.Run(args) } diff --git a/cmd/athome/server.go b/cmd/athome/server.go deleted file mode 100644 index 49dd23d7..00000000 --- a/cmd/athome/server.go +++ /dev/null @@ -1,218 +0,0 @@ -package main - -import ( - "context" - "fmt" - "io/fs" - "net/http" - "os" - "strings" - "embed" - - comatproto "github.com/bluesky-social/indigo/api/atproto" - appbsky "github.com/bluesky-social/indigo/api/bsky" - cliutil "github.com/bluesky-social/indigo/cmd/gosky/util" - "github.com/bluesky-social/indigo/xrpc" - - "github.com/flosch/pongo2/v6" - "github.com/labstack/echo/v4" - "github.com/labstack/echo/v4/middleware" - "github.com/urfave/cli/v2" -) - -//go:embed static/* -var StaticFS embed.FS - -type Server struct { - xrpcc *xrpc.Client -} - -func serve(cctx *cli.Context) error { - debug := cctx.Bool("debug") - httpAddress := cctx.String("http-address") - pdsHost := cctx.String("pds-host") - atpHandle := cctx.String("handle") - atpPassword := cctx.String("password") - - // create a new session - // TODO: does this work with no auth at all? - xrpcc := &xrpc.Client{ - Client: cliutil.NewHttpClient(), - Host: pdsHost, - Auth: &xrpc.AuthInfo{ - Handle: atpHandle, - }, - } - - auth, err := comatproto.ServerCreateSession(context.TODO(), xrpcc, &comatproto.ServerCreateSession_Input{ - Identifier: xrpcc.Auth.Handle, - Password: atpPassword, - }) - if err != nil { - return err - } - xrpcc.Auth.AccessJwt = auth.AccessJwt - xrpcc.Auth.RefreshJwt = auth.RefreshJwt - xrpcc.Auth.Did = auth.Did - xrpcc.Auth.Handle = auth.Handle - - server := Server{xrpcc} - - staticHandler := http.FileServer(func() http.FileSystem { - if debug { - return http.FS(os.DirFS("static")) - } - fsys, err := fs.Sub(StaticFS, "static") - if err != nil { - log.Fatal(err) - } - return http.FS(fsys) - }()) - - e := echo.New() - e.HideBanner = true - // SECURITY: Do not modify without due consideration. - e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ - ContentTypeNosniff: "nosniff", - XFrameOptions: "SAMEORIGIN", - HSTSMaxAge: 31536000, // 365 days - // TODO: - // ContentSecurityPolicy - // XSSProtection - })) - e.Use(middleware.LoggerWithConfig(middleware.LoggerConfig{ - // Don't log requests for static content. - Skipper: func(c echo.Context) bool { - return strings.HasPrefix(c.Request().URL.Path, "/static") - }, - Format: "method=${method} path=${uri} status=${status} latency=${latency_human}\n", - })) - e.Renderer = NewRenderer("templates/", &TemplateFS, debug) - e.HTTPErrorHandler = customHTTPErrorHandler - - // redirect trailing slash to non-trailing slash. - // all of our current endpoints have no trailing slash. - e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{ - RedirectCode: http.StatusFound, - })) - - // basic static routes - e.GET("/robots.txt", echo.WrapHandler(staticHandler)) - e.GET("/favicon.ico", echo.WrapHandler(staticHandler)) - e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler))) - e.GET("/", server.WebHome) - - // generic routes - //e.GET("/search", server.WebGeneric) - //e.GET("/support", server.WebGeneric) - //e.GET("/support/privacy", server.WebGeneric) - //e.GET("/support/tos", server.WebGeneric) - //e.GET("/support/community-guidelines", server.WebGeneric) - //e.GET("/support/copyright", server.WebGeneric) - - // profile endpoints; only first populates info - e.GET("/profile/:handle", server.WebProfile) - //e.GET("/profile/:handle/repo.car.gz", server.WebProfile) - //e.GET("/profile/:handle/follows", server.WebGeneric) - //e.GET("/profile/:handle/followers", server.WebGeneric) - - // post endpoints; only first populates info - e.GET("/profile/:handle/post/:rkey", server.WebPost) - //e.GET("/profile/:handle/post/:rkey/liked-by", server.WebGeneric) - //e.GET("/profile/:handle/post/:rkey/reposted-by", server.WebGeneric) - - // feeds - e.GET("/feed/:name", server.WebFeed) - - // redirect - //e.GET("/at://:account", server.WebAccountURI) - //e.GET("/at://:account/:nsid/:rkey", server.WebRecordURI) - - log.Infow("starting server", "bind", httpAddress) - return e.Start(httpAddress) -} - -func customHTTPErrorHandler(err error, c echo.Context) { - code := http.StatusInternalServerError - if he, ok := err.(*echo.HTTPError); ok { - code = he.Code - } - c.Logger().Error(err) - data := pongo2.Context{ - "statusCode": code, - } - c.Render(code, "error.html", data) -} - -func (srv *Server) WebHome(c echo.Context) error { - data := pongo2.Context{} - return c.Render(http.StatusOK, "home.html", data) -} - -func (srv *Server) WebPost(c echo.Context) error { - data := pongo2.Context{} - handle := c.Param("handle") - rkey := c.Param("rkey") - // sanity check argument - if len(handle) > 4 && len(handle) < 128 && len(rkey) > 0 { - ctx := c.Request().Context() - // requires two fetches: first fetch profile (!) - pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) - if err != nil { - log.Warnf("failed to fetch handle: %s\t%v", handle, err) - // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) - } else { - did := pv.Did - data["did"] = did - - // then fetch the post thread (with extra context) - uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) - tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, uri) - if err != nil { - log.Warnf("failed to fetch post: %s\t%v", uri, err) - // TODO: only if "not found" - return echo.NewHTTPError(404, "post not found: %s", handle) - } else { - req := c.Request() - postView := tpv.Thread.FeedDefs_ThreadViewPost.Post - data["postView"] = postView - data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) - if postView.Embed != nil && postView.Embed.EmbedImages_View != nil { - data["imgThumbUrl"] = postView.Embed.EmbedImages_View.Images[0].Thumb - } - } - } - - } - return c.Render(http.StatusOK, "post.html", data) -} - -func (srv *Server) WebProfile(c echo.Context) error { - data := pongo2.Context{} - handle := c.Param("handle") - // sanity check argument - if len(handle) > 4 && len(handle) < 128 { - ctx := c.Request().Context() - pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) - if err != nil { - log.Warnw("failed to fetch handle", "handle", handle, "err", err) - // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) - } else { - req := c.Request() - data["profileView"] = pv - data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) - } - af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle, "", 100) - if err != nil { - log.Warnw("failed to fetch author feed", "handle", handle, "err", err) - // TODO: show some error? - } else { - data["authorFeed"] = af.Feed - //log.Warnw("author feed", "feed", af.Feed) - } - } - - return c.Render(http.StatusOK, "profile.html", data) -} diff --git a/cmd/athome/service.go b/cmd/athome/service.go new file mode 100644 index 00000000..5567c295 --- /dev/null +++ b/cmd/athome/service.go @@ -0,0 +1,202 @@ +package main + +import ( + "context" + "io/fs" + "net/http" + "os" + "time" + "embed" + "os/signal" + "errors" + "syscall" + + "github.com/bluesky-social/indigo/xrpc" + "github.com/bluesky-social/indigo/util" + + "github.com/urfave/cli/v2" + "github.com/flosch/pongo2/v6" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/labstack/echo-contrib/echoprometheus" + slogecho "github.com/samber/slog-echo" +) + +//go:embed static/* +var StaticFS embed.FS + +type Server struct { + echo *echo.Echo + httpd *http.Server + xrpcc *xrpc.Client +} + +func serve(cctx *cli.Context) error { + debug := cctx.Bool("debug") + httpAddress := cctx.String("bind") + appviewHost := cctx.String("appview-host") + + xrpcc := &xrpc.Client{ + Client: util.RobustHTTPClient(), + Host: appviewHost, + // Headers: version + } + e := echo.New() + + // httpd + var ( + httpTimeout = 1 * time.Minute + httpMaxHeaderBytes = 1 * (1024 * 1024) + ) + + srv := &Server{ + echo: e, + xrpcc: xrpcc, + } + srv.httpd = &http.Server{ + Handler: srv, + Addr: httpAddress, + WriteTimeout: httpTimeout, + ReadTimeout: httpTimeout, + MaxHeaderBytes: httpMaxHeaderBytes, + } + + e.HideBanner = true + e.Use(slogecho.New(slog)) + e.Use(middleware.Recover()) + e.Use(echoprometheus.NewMiddleware("athome")) + e.Use(middleware.BodyLimit("64M")) + e.HTTPErrorHandler = srv.errorHandler + e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ + ContentTypeNosniff: "nosniff", + XFrameOptions: "SAMEORIGIN", + HSTSMaxAge: 31536000, // 365 days + // TODO: + // ContentSecurityPolicy + // XSSProtection + })) + + // redirect trailing slash to non-trailing slash. + // all of our current endpoints have no trailing slash. + e.Use(middleware.RemoveTrailingSlashWithConfig(middleware.TrailingSlashConfig{ + RedirectCode: http.StatusFound, + })) + + staticHandler := http.FileServer(func() http.FileSystem { + if debug { + return http.FS(os.DirFS("static")) + } + fsys, err := fs.Sub(StaticFS, "static") + if err != nil { + slog.Error("static template error", "err", err) + os.Exit(-1) + } + return http.FS(fsys) + }()) + + e.Renderer = NewRenderer("templates/", &TemplateFS, debug) + e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler))) + + e.GET("/_health", srv.HandleHealthCheck) + e.GET("/metrics", echoprometheus.NewHandler()) + + // basic static routes + e.GET("/robots.txt", echo.WrapHandler(staticHandler)) + e.GET("/favicon.ico", echo.WrapHandler(staticHandler)) + e.GET("/", srv.WebHome) + + // generic routes + //e.GET("/search", srv.WebGeneric) + //e.GET("/support", srv.WebGeneric) + //e.GET("/support/privacy", srv.WebGeneric) + //e.GET("/support/tos", srv.WebGeneric) + //e.GET("/support/community-guidelines", srv.WebGeneric) + //e.GET("/support/copyright", srv.WebGeneric) + + // profile endpoints; only first populates info + e.GET("/profile/:handle", srv.WebProfile) + //e.GET("/profile/:handle/repo.car.gz", srv.WebProfile) + //e.GET("/profile/:handle/follows", srv.WebGeneric) + //e.GET("/profile/:handle/followers", srv.WebGeneric) + + // post endpoints; only first populates info + e.GET("/profile/:handle/post/:rkey", srv.WebPost) + //e.GET("/profile/:handle/post/:rkey/liked-by", srv.WebGeneric) + //e.GET("/profile/:handle/post/:rkey/reposted-by", srv.WebGeneric) + + // feeds + //e.GET("/feed/:name", srv.WebFeed) + + // redirect + //e.GET("/at://:account", srv.WebAccountURI) + //e.GET("/at://:account/:nsid/:rkey", srv.WebRecordURI) + + // Start the server + slog.Info("starting server", "bind", httpAddress) + go func() { + if err := srv.httpd.ListenAndServe(); err != nil { + if !errors.Is(err, http.ErrServerClosed) { + slog.Error("HTTP server shutting down unexpectedly", "err", err) + } + } + }() + + // Wait for a signal to exit. + slog.Info("registering OS exit signal handler") + quit := make(chan struct{}) + exitSignals := make(chan os.Signal, 1) + signal.Notify(exitSignals, syscall.SIGINT, syscall.SIGTERM) + go func() { + sig := <-exitSignals + slog.Info("received OS exit signal", "signal", sig) + + // Shut down the HTTP server + if err := srv.Shutdown(); err != nil { + slog.Error("HTTP server shutdown error", "err", err) + } + + // Trigger the return that causes an exit. + close(quit) + }() + <-quit + slog.Info("graceful shutdown complete") + return nil +} + +type GenericStatus struct { + Daemon string `json:"daemon"` + Status string `json:"status"` + Message string `json:"msg,omitempty"` +} + +func (srv *Server) errorHandler(err error, c echo.Context) { + code := http.StatusInternalServerError + if he, ok := err.(*echo.HTTPError); ok { + code = he.Code + } + if code >= 500 { + slog.Warn("abyss-http-internal-error", "err", err) + } + data := pongo2.Context{ + "statusCode": code, + } + c.Render(code, "error.html", data) +} + +func (srv *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + srv.echo.ServeHTTP(rw, req) +} + +func (srv *Server) Shutdown() error { + slog.Info("shutting down") + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + return srv.httpd.Shutdown(ctx) +} + +func (s *Server) HandleHealthCheck(c echo.Context) error { + return c.JSON(200, GenericStatus{Status: "ok", Daemon: "abyss"}) +} + diff --git a/go.mod b/go.mod index 8869d5b0..0223f3c8 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,9 @@ require ( contrib.go.opencensus.io/exporter/prometheus v0.4.2 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/brianvoe/gofakeit/v6 v6.20.2 + github.com/carlmjohnson/versioninfo v0.22.5 github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49 + github.com/flosch/pongo2/v6 v6.0.0 github.com/goccy/go-json v0.10.2 github.com/gocql/gocql v1.6.0 github.com/golang-jwt/jwt v3.2.2+incompatible diff --git a/go.sum b/go.sum index 6f6f70ac..cbc69fec 100644 --- a/go.sum +++ b/go.sum @@ -73,6 +73,8 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/brianvoe/gofakeit/v6 v6.20.2 h1:FLloufuC7NcbHqDzVQ42CG9AKryS1gAGCRt8nQRsW+Y= github.com/brianvoe/gofakeit/v6 v6.20.2/go.mod h1:Ow6qC71xtwm79anlwKRlWZW6zVq9D2XHE4QSSMP/rU8= +github.com/carlmjohnson/versioninfo v0.22.5 h1:O00sjOLUAFxYQjlN/bzYTuZiS0y6fWDQjMRvwtKgwwc= +github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPviHuSF+cUtLjm2WSf8= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -117,6 +119,8 @@ github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go. github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flosch/pongo2/v6 v6.0.0 h1:lsGru8IAzHgIAw6H2m4PCyleO58I40ow6apih0WprMU= +github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT5LJhsHEU= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -- 2.51.2 From 0930570182d5409319b78a336e541ea709949270 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 22:28:47 -0700 Subject: [PATCH 124/192] athome: cleanups, better display --- cmd/athome/handlers.go | 197 +++++++++++++++++++------- cmd/athome/main.go | 21 ++- cmd/athome/service.go | 73 ++++------ cmd/athome/templates/base.html | 12 +- cmd/athome/templates/error.html | 2 +- cmd/athome/templates/feed_macros.html | 99 ++++++------- cmd/athome/templates/home.html | 20 --- cmd/athome/templates/post.html | 41 +++--- cmd/athome/templates/profile.html | 12 +- 9 files changed, 268 insertions(+), 209 deletions(-) delete mode 100644 cmd/athome/templates/home.html diff --git a/cmd/athome/handlers.go b/cmd/athome/handlers.go index e81cd119..287ea382 100644 --- a/cmd/athome/handlers.go +++ b/cmd/athome/handlers.go @@ -3,83 +3,172 @@ package main import ( "fmt" "net/http" + "strings" appbsky "github.com/bluesky-social/indigo/api/bsky" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/flosch/pongo2/v6" "github.com/labstack/echo/v4" ) +func (srv *Server) reqHandle(c echo.Context) syntax.Handle { + host := c.Request().Host + host = strings.SplitN(host, ":", 2)[0] + handle, err := syntax.ParseHandle(host) + if err != nil { + slog.Warn("host is not a valid handle, fallback to default", "host", host) + handle = srv.defaultHandle + } + return handle +} + func (srv *Server) WebHome(c echo.Context) error { - data := pongo2.Context{} - return c.Render(http.StatusOK, "home.html", data) + return c.Redirect(http.StatusFound, "/bsky") +} + +func (srv *Server) WebRepoCar(c echo.Context) error { + handle := srv.reqHandle(c) + ident, err := srv.dir.LookupHandle(c.Request().Context(), handle) + if err != nil { + return err + } + return c.Redirect(http.StatusFound, ident.PDSEndpoint()+"/xrpc/com.atproto.sync.getRepo?did="+ident.DID.String()) } func (srv *Server) WebPost(c echo.Context) error { + ctx := c.Request().Context() + req := c.Request() data := pongo2.Context{} - handle := c.Param("handle") + handle := srv.reqHandle(c) + // TODO: parse rkey rkey := c.Param("rkey") - // sanity check argument - if len(handle) > 4 && len(handle) < 128 && len(rkey) > 0 { - ctx := c.Request().Context() - // requires two fetches: first fetch profile (!) - pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) - if err != nil { - slog.Warn("failed to fetch handle", "handle", handle, "err", err) - // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) - } else { - did := pv.Did - data["did"] = did - - // then fetch the post thread (with extra context) - uri := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) - // TODO: more of thread? - tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 1, 1, uri) - if err != nil { - slog.Warn("failed to fetch post", "uri", uri, "err", err) - // TODO: only if "not found" - return echo.NewHTTPError(404, "post not found: %s", handle) - } else { - req := c.Request() - postView := tpv.Thread.FeedDefs_ThreadViewPost.Post - data["postView"] = postView - data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) - if postView.Embed != nil && postView.Embed.EmbedImages_View != nil { - data["imgThumbUrl"] = postView.Embed.EmbedImages_View.Images[0].Thumb - } - } - } + // requires two fetches: first fetch profile (!) + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle.String()) + if err != nil { + slog.Warn("failed to fetch handle", "handle", handle, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) } + did := pv.Did + data["did"] = did + + // then fetch the post thread (with extra context) + aturi := fmt.Sprintf("at://%s/app.bsky.feed.post/%s", did, rkey) + tpv, err := appbsky.FeedGetPostThread(ctx, srv.xrpcc, 8, 8, aturi) + if err != nil { + slog.Warn("failed to fetch post", "aturi", aturi, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "post not found: %s", handle) + } + data["postView"] = tpv.Thread.FeedDefs_ThreadViewPost + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) return c.Render(http.StatusOK, "post.html", data) } func (srv *Server) WebProfile(c echo.Context) error { + ctx := c.Request().Context() data := pongo2.Context{} - handle := c.Param("handle") - // sanity check argument - if len(handle) > 4 && len(handle) < 128 { - ctx := c.Request().Context() - pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle) - if err != nil { - slog.Warn("failed to fetch handle", "handle", handle, "err", err) - // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) - } else { - req := c.Request() - data["profileView"] = pv - data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + handle := srv.reqHandle(c) + + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle.String()) + if err != nil { + slog.Warn("failed to fetch handle", "handle", handle, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) + } else { + req := c.Request() + data["profileView"] = pv + data["requestURI"] = fmt.Sprintf("https://%s%s", req.Host, req.URL.Path) + } + did := pv.Did + data["did"] = did + + af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle.String(), "", "", 100) + if err != nil { + slog.Warn("failed to fetch author feed", "handle", handle, "err", err) + // TODO: show some error? + } else { + data["authorFeed"] = af.Feed + //slog.Warn("author feed", "feed", af.Feed) + } + + return c.Render(http.StatusOK, "profile.html", data) +} + +// https://medium.com/@etiennerouzeaud/a-rss-feed-valid-in-go-edfc22e410c7 +type Item struct { + Title string `xml:"title"` + Link string `xml:"link"` + Description string `xml:"description"` + PubDate string `xml:"pubDate"` +} + +type rss struct { + Version string `xml:"version,attr"` + Description string `xml:"channel>description"` + Link string `xml:"channel>link"` + Title string `xml:"channel>title"` + + Item []Item `xml:"channel>item"` +} + +func (srv *Server) WebRepoRSS(c echo.Context) error { + ctx := c.Request().Context() + handle := srv.reqHandle(c) + + pv, err := appbsky.ActorGetProfile(ctx, srv.xrpcc, handle.String()) + if err != nil { + slog.Warn("failed to fetch handle", "handle", handle, "err", err) + // TODO: only if "not found" + return echo.NewHTTPError(404, "handle not found: %s", handle) + //return err + } + + af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle.String(), "", "", 30) + if err != nil { + slog.Warn("failed to fetch author feed", "handle", handle, "err", err) + return err + } + + posts := []Item{} + for _, p := range af.Feed { + // only include own posts in RSS + if p.Post.Author.Did != pv.Did { + continue } - af, err := appbsky.FeedGetAuthorFeed(ctx, srv.xrpcc, handle, "", "", 100) + aturi, err := syntax.ParseATURI(p.Post.Uri) if err != nil { - slog.Warn("failed to fetch author feed", "handle", handle, "err", err) - // TODO: show some error? - } else { - data["authorFeed"] = af.Feed - //slog.Warn("author feed", "feed", af.Feed) + return err + } + rec := p.Post.Record.Val.(*appbsky.FeedPost) + // only top-level posts in RSS + if rec.Reply != nil { + continue } + posts = append(posts, Item{ + Title: "@" + handle.String() + " post", + Link: fmt.Sprintf("https://%s/bsky/post/%s", handle, aturi.RecordKey().String()), + Description: rec.Text, + PubDate: rec.CreatedAt, + }) } - return c.Render(http.StatusOK, "profile.html", data) + title := "@" + handle.String() + if pv.DisplayName != nil { + title = title + " - " + *pv.DisplayName + } + desc := "" + if pv.Description != nil { + desc = *pv.Description + } + feed := &rss{ + Version: "2.0", + Description: desc, + Link: fmt.Sprintf("https://%s/bsky", handle.String()), + Title: title, + Item: posts, + } + return c.XML(http.StatusOK, feed) } diff --git a/cmd/athome/main.go b/cmd/athome/main.go index cd0619ba..eb791c98 100644 --- a/cmd/athome/main.go +++ b/cmd/athome/main.go @@ -1,9 +1,9 @@ package main import ( + "fmt" slogging "log/slog" "os" - "fmt" "github.com/carlmjohnson/versioninfo" "github.com/urfave/cli/v2" @@ -11,10 +11,9 @@ import ( _ "github.com/joho/godotenv/autoload" ) - var ( - slog = slogging.New(slogging.NewJSONHandler(os.Stdout, nil)) - version = versioninfo.Short() + slog = slogging.New(slogging.NewJSONHandler(os.Stdout, nil)) + version = versioninfo.Short() ) func main() { @@ -60,13 +59,13 @@ func run(args []string) error { }, }, &cli.Command{ - Name: "version", - Usage: "print version", - Action: func(cctx *cli.Context) error { - fmt.Println(version) - return nil - }, - }, + Name: "version", + Usage: "print version", + Action: func(cctx *cli.Context) error { + fmt.Println(version) + return nil + }, + }, } return app.Run(args) diff --git a/cmd/athome/service.go b/cmd/athome/service.go index 5567c295..739b75eb 100644 --- a/cmd/athome/service.go +++ b/cmd/athome/service.go @@ -2,33 +2,37 @@ package main import ( "context" + "embed" + "errors" "io/fs" "net/http" "os" - "time" - "embed" "os/signal" - "errors" "syscall" + "time" - "github.com/bluesky-social/indigo/xrpc" + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/util" + "github.com/bluesky-social/indigo/xrpc" - "github.com/urfave/cli/v2" "github.com/flosch/pongo2/v6" + "github.com/labstack/echo-contrib/echoprometheus" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" - "github.com/labstack/echo-contrib/echoprometheus" slogecho "github.com/samber/slog-echo" + "github.com/urfave/cli/v2" ) //go:embed static/* var StaticFS embed.FS type Server struct { - echo *echo.Echo - httpd *http.Server - xrpcc *xrpc.Client + echo *echo.Echo + httpd *http.Server + dir identity.Directory // TODO: unused? + xrpcc *xrpc.Client + defaultHandle syntax.Handle } func serve(cctx *cli.Context) error { @@ -36,6 +40,11 @@ func serve(cctx *cli.Context) error { httpAddress := cctx.String("bind") appviewHost := cctx.String("appview-host") + dh, err := syntax.ParseHandle("atproto.com") + if err != nil { + return err + } + xrpcc := &xrpc.Client{ Client: util.RobustHTTPClient(), Host: appviewHost, @@ -50,8 +59,10 @@ func serve(cctx *cli.Context) error { ) srv := &Server{ - echo: e, - xrpcc: xrpcc, + echo: e, + xrpcc: xrpcc, + dir: identity.DefaultDirectory(), + defaultHandle: dh, } srv.httpd = &http.Server{ Handler: srv, @@ -67,6 +78,7 @@ func serve(cctx *cli.Context) error { e.Use(echoprometheus.NewMiddleware("athome")) e.Use(middleware.BodyLimit("64M")) e.HTTPErrorHandler = srv.errorHandler + e.Renderer = NewRenderer("templates/", &TemplateFS, debug) e.Use(middleware.SecureWithConfig(middleware.SecureConfig{ ContentTypeNosniff: "nosniff", XFrameOptions: "SAMEORIGIN", @@ -94,42 +106,20 @@ func serve(cctx *cli.Context) error { return http.FS(fsys) }()) - e.Renderer = NewRenderer("templates/", &TemplateFS, debug) e.GET("/static/*", echo.WrapHandler(http.StripPrefix("/static/", staticHandler))) - e.GET("/_health", srv.HandleHealthCheck) e.GET("/metrics", echoprometheus.NewHandler()) // basic static routes e.GET("/robots.txt", echo.WrapHandler(staticHandler)) e.GET("/favicon.ico", echo.WrapHandler(staticHandler)) - e.GET("/", srv.WebHome) - // generic routes - //e.GET("/search", srv.WebGeneric) - //e.GET("/support", srv.WebGeneric) - //e.GET("/support/privacy", srv.WebGeneric) - //e.GET("/support/tos", srv.WebGeneric) - //e.GET("/support/community-guidelines", srv.WebGeneric) - //e.GET("/support/copyright", srv.WebGeneric) - - // profile endpoints; only first populates info - e.GET("/profile/:handle", srv.WebProfile) - //e.GET("/profile/:handle/repo.car.gz", srv.WebProfile) - //e.GET("/profile/:handle/follows", srv.WebGeneric) - //e.GET("/profile/:handle/followers", srv.WebGeneric) - - // post endpoints; only first populates info - e.GET("/profile/:handle/post/:rkey", srv.WebPost) - //e.GET("/profile/:handle/post/:rkey/liked-by", srv.WebGeneric) - //e.GET("/profile/:handle/post/:rkey/reposted-by", srv.WebGeneric) - - // feeds - //e.GET("/feed/:name", srv.WebFeed) - - // redirect - //e.GET("/at://:account", srv.WebAccountURI) - //e.GET("/at://:account/:nsid/:rkey", srv.WebRecordURI) + // actual content + e.GET("/", srv.WebHome) + e.GET("/bsky", srv.WebProfile) + e.GET("/bsky/post/:rkey", srv.WebPost) + e.GET("/bsky/repo.car", srv.WebRepoCar) + e.GET("/bsky/rss.xml", srv.WebRepoRSS) // Start the server slog.Info("starting server", "bind", httpAddress) @@ -175,7 +165,7 @@ func (srv *Server) errorHandler(err error, c echo.Context) { code = he.Code } if code >= 500 { - slog.Warn("abyss-http-internal-error", "err", err) + slog.Warn("athome-http-internal-error", "err", err) } data := pongo2.Context{ "statusCode": code, @@ -197,6 +187,5 @@ func (srv *Server) Shutdown() error { } func (s *Server) HandleHealthCheck(c echo.Context) error { - return c.JSON(200, GenericStatus{Status: "ok", Daemon: "abyss"}) + return c.JSON(200, GenericStatus{Status: "ok", Daemon: "athome"}) } - diff --git a/cmd/athome/templates/base.html b/cmd/athome/templates/base.html index a452d128..75dd3ba0 100644 --- a/cmd/athome/templates/base.html +++ b/cmd/athome/templates/base.html @@ -29,7 +29,7 @@ {% block html_head_extra -%}{%- endblock %} - + {%- block body_all %} @@ -37,12 +37,10 @@
-

Bluesky

- What's Hot - Hell Thread - Prod bnewbold - Staging bnewbold - Staging paul +

{%- block sidebar_title -%}Bluesky{%- endblock -%}

+ Profile + repo.car + RSS
diff --git a/cmd/athome/templates/error.html b/cmd/athome/templates/error.html index 4944f5d4..53d0a118 100644 --- a/cmd/athome/templates/error.html +++ b/cmd/athome/templates/error.html @@ -7,6 +7,6 @@

{{ statusCode }}

Error!

-

Sorry about that! Our Status Page might have more context. +

Sorry about that! The Bluesky Status Page might have more context.

{% endblock %} diff --git a/cmd/athome/templates/feed_macros.html b/cmd/athome/templates/feed_macros.html index 3b990077..6d444bf1 100644 --- a/cmd/athome/templates/feed_macros.html +++ b/cmd/athome/templates/feed_macros.html @@ -1,6 +1,10 @@ -{% macro feed_post(feedItem) export %} +{% macro feed_post(feedItem, selfDID, primary) export %} +{% if primary %} +
+{% else %}
+{% endif %}
{% if feedItem.Post.Author.Avatar %} @@ -8,23 +12,33 @@ {% endif %}
-
+
{% if feedItem.Reason %} {{ feedItem.Reason.FeedDefs_ReasonRepost }} {% endif %}
- + {% if feedItem.Post.Author.Did == selfDID %} + + {% else %} + + {% endif %} {% if feedItem.Post.Author.DisplayName %} {{ feedItem.Post.Author.DisplayName }} + + {% else %} + {% endif %} - @{{ feedItem.Post.Author.Handle }} + @{{ feedItem.Post.Author.Handle }} -
{# TODO: relative time#} {# TODO: parse and fix link (custom filter?) #} - {{ feedItem.Post.IndexedAt }} + {% if feedItem.Post.Author.Did == selfDID %} + {{ feedItem.Post.IndexedAt }} + {% else %} + {{ feedItem.Post.IndexedAt }} + {% endif %}
@@ -50,57 +64,30 @@
-{% endmacro %} -{% macro primary_post(feedItem) export %} -
-
- {% if feedItem.Post.Author.Avatar %} - - {% else %} - - {% endif %} -
-
- {% if feedItem.Reason %} - {{ feedItem.Reason.FeedDefs_ReasonRepost }} - {% endif %} -
- - {% if feedItem.Post.Author.DisplayName %} - {{ feedItem.Post.Author.DisplayName }} - {% endif %} - @{{ feedItem.Post.Author.Handle }} - +{% if primary %} + +{% endif %} +{% endmacro %} +{% macro thread_parents(post, selfDID, primary) export %} +{% if post.Parent %} + {{ thread_parents(post.Parent.FeedDefs_ThreadViewPost, selfDID, false) }} +
+{% endif %} +{{ feed_post(post, selfDID, primary) }} +{% endmacro %} -
- {# TODO: relative time#} - {# TODO: parse and fix link (custom filter?) #} - {{ feedItem.Post.IndexedAt }} -
-
-
- {{ feedItem.Post.Record.Val.Text }} - {% if feedItem.Post.Embed and feedItem.Post.Embed.EmbedImages_View %} -
- {% for image in feedItem.Post.Embed.EmbedImages_View.Images %} -
-
- - {{ image.Alt }} - -
-
- {% endfor %} -
- {% endif %} -
-
- - - -
-
-
+{% macro thread_children(post, selfDID) export %} +{% for child in post.Replies %} +
+ {{ feed_post(child.FeedDefs_ThreadViewPost, selfDID) }} + {{ thread_children(child.FeedDefs_ThreadViewPost, selfDID) }} +{% endfor %} {% endmacro %} diff --git a/cmd/athome/templates/home.html b/cmd/athome/templates/home.html deleted file mode 100644 index 9240859c..00000000 --- a/cmd/athome/templates/home.html +++ /dev/null @@ -1,20 +0,0 @@ -{% extends "base.html" %} - -{% block head_title %}Bluesky{% endblock %} - -{% block html_head_extra -%} - - - - - - - -{%- endblock %} - -{% block main_content %} -
-

See what’s next

-

Bluesky Social

-
-{% endblock %} diff --git a/cmd/athome/templates/post.html b/cmd/athome/templates/post.html index 694e608b..ee4db544 100644 --- a/cmd/athome/templates/post.html +++ b/cmd/athome/templates/post.html @@ -1,46 +1,55 @@ {% extends "base.html" %} {% block head_title %} -{%- if postView -%} - @{{ postView.Author.Handle }} on Bluesky +{%- if postView.Post -%} + @{{ postView.Post.Author.Handle }} on Bluesky +{%- else -%} + Bluesky +{%- endif -%} +{% endblock %} + +{% block sidebar_title %} +{%- if postView.Post -%} + {{ postView.Post.Author.Handle }} {%- else -%} Bluesky {%- endif -%} {% endblock %} {% block html_head_extra -%} -{%- if postView -%} +{%- if postView.Post -%} {%- if requestURI %} {% endif -%} - {%- if postView.Author.DisplayName %} - + {%- if postView.Post.Author.DisplayName %} + {% else %} - + {% endif -%} - {%- if postView.Record.Val.Text %} - - + {%- if postView.Post.Record.Val.Text %} + + {% endif -%} {%- if imgThumbUrl %} - {%- elif postView.Author.Avatar %} + {%- elif postView.Post.Author.Avatar %} {# Don't use avatar image in cards; usually looks bad #} {% endif %} - + {% endif -%} {%- endblock %} {% block main_content %} -

Post

-

{{ postView.Author.DisplayName }}

-

{{ postView.Author.Handle }}

-

{{ postView.Author.Did }}

-

{{ postView.Record.Val.Text }}

+ {% import "feed_macros.html" feed_post, thread_parents, thread_children %} +
+
+ {{ thread_parents(postView, did, true) }} + {{ thread_children(postView) }} +
{%- endblock %} diff --git a/cmd/athome/templates/profile.html b/cmd/athome/templates/profile.html index 637c190f..7110bf0e 100644 --- a/cmd/athome/templates/profile.html +++ b/cmd/athome/templates/profile.html @@ -8,6 +8,14 @@ {%- endif -%} {% endblock %} +{% block sidebar_title %} +{%- if profileView -%} + {{ profileView.Handle }} +{%- else -%} + Bluesky +{%- endif -%} +{% endblock %} + {% block html_head_extra -%} {%- if profileView -%} @@ -58,9 +66,9 @@

{{ profileView.Description }}

-
+
{% for feedItem in authorFeed %} - {{ feed_post(feedItem) }} + {{ feed_post(feedItem, did) }}
{% endfor %}
-- 2.51.2 From 07f8f35e236c58a3070c9808638b5c83d6b88340 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 4 Oct 2023 00:12:49 -0700 Subject: [PATCH 125/192] athome: fix error formatting --- cmd/athome/handlers.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/athome/handlers.go b/cmd/athome/handlers.go index 287ea382..199f37e7 100644 --- a/cmd/athome/handlers.go +++ b/cmd/athome/handlers.go @@ -49,7 +49,7 @@ func (srv *Server) WebPost(c echo.Context) error { if err != nil { slog.Warn("failed to fetch handle", "handle", handle, "err", err) // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) + return echo.NewHTTPError(404, fmt.Sprintf("handle not found: %s", handle)) } did := pv.Did data["did"] = did @@ -76,7 +76,7 @@ func (srv *Server) WebProfile(c echo.Context) error { if err != nil { slog.Warn("failed to fetch handle", "handle", handle, "err", err) // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) + return echo.NewHTTPError(404, fmt.Sprintf("handle not found: %s", handle)) } else { req := c.Request() data["profileView"] = pv @@ -122,7 +122,7 @@ func (srv *Server) WebRepoRSS(c echo.Context) error { if err != nil { slog.Warn("failed to fetch handle", "handle", handle, "err", err) // TODO: only if "not found" - return echo.NewHTTPError(404, "handle not found: %s", handle) + return echo.NewHTTPError(404, fmt.Sprintf("handle not found: %s", handle)) //return err } -- 2.51.2 From 4a21a0eace2c3e199fec2b10a6de0b3819c17571 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 4 Oct 2023 01:09:37 -0700 Subject: [PATCH 126/192] athome: update README --- cmd/athome/README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/cmd/athome/README.md b/cmd/athome/README.md index 6bb954b2..5f5240c4 100644 --- a/cmd/athome/README.md +++ b/cmd/athome/README.md @@ -12,3 +12,57 @@ public web interface at home: 2. point one or more handle domains to it (CNAME or reverse proxy) 3. serves up profile and feed for that account only 4. fetches data from public bsky app view API + + +## Running athome + +The recommended way to run `athome` is behind a `caddy` HTTPS server which does automatic on-demand SSL certificate registration (using Let's Encrypt). + +Build and run `athome`: + + go build ./cmd/athome + + # will listen on :8200 by default + ./athome serve + +Create a `Caddyfile`: + +``` +{ + on_demand_tls { + interval 1h + burst 8 + } +} + +:443 { + reverse_proxy localhost:8200 + tls YOUREMAIL@example.com { + on_demand + } +} +``` + +Run `caddy`: + + caddy run + + +## Configuring a Handle + +The easiest way, if there is no existing web service on the handle domain, is to get the handle resolution working with the DNS TXT record option, then point the domain itself to a `athome` service using an A/AAAA or CNAME record. + +If there is an existing web service (eg, a blog), then handle resolution can be set up using either the DNS TXT mechanism or HTTP `/.well-known/` mechanism. Then HTTP proxy paths starting `/bsky` to an `athome` service. + +Here is an nginx config snippet demonstrating HTTP proxying: + +``` +location /bsky { + // in theory https:// should work, on default port? + proxy_pass http://athome.example.com:8200; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto https; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; +} +``` -- 2.51.2 From 00740f960e956c8c88a1b2e36441172f9efb433c Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 6 Oct 2023 14:55:44 -0700 Subject: [PATCH 127/192] athome: disclaim in readme, add a gitignore --- .gitignore | 5 +++-- cmd/athome/README.md | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index a3a92e50..79926f84 100644 --- a/.gitignore +++ b/.gitignore @@ -21,16 +21,17 @@ data/ test-coverage.out # executables +/athome /beemo /bigsky /fakermaker /gosky +/labelmaker /laputa /lexgen -/stress -/labelmaker /palomar /sonar-cli +/stress /supercollider # Don't ignore this file itself, or other specific dotfiles diff --git a/cmd/athome/README.md b/cmd/athome/README.md index 5f5240c4..f4df2480 100644 --- a/cmd/athome/README.md +++ b/cmd/athome/README.md @@ -13,6 +13,10 @@ public web interface at home: 3. serves up profile and feed for that account only 4. fetches data from public bsky app view API +⚠️ This is a fun little proof-of-concept ⚠️ + +Not all post features are rendered, has not been hardened against clever Unicode tricks, etc. + ## Running athome -- 2.51.2 From 61c1440145569c287c68749a55c3a55dd9a8b46c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 9 Oct 2023 11:18:44 -0700 Subject: [PATCH 128/192] combine all staleRefs for a given operation into a single record --- carstore/bs.go | 121 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 83 insertions(+), 38 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index 56839556..06ea9e02 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -94,9 +94,45 @@ type blockRef struct { } type staleRef struct { - ID uint `gorm:"primarykey"` - Cid models.DbCID - Usr models.Uid `gorm:"index"` + ID uint `gorm:"primarykey"` + Cid *models.DbCID + Cids []byte + Usr models.Uid `gorm:"index"` +} + +func (sr *staleRef) getCids() ([]cid.Cid, error) { + if sr.Cid != nil { + return []cid.Cid{sr.Cid.CID}, nil + } + + return unpackCids(sr.Cids) +} + +func packCids(cids []cid.Cid) []byte { + buf := new(bytes.Buffer) + for _, c := range cids { + buf.Write(c.Bytes()) + } + + return buf.Bytes() +} + +func unpackCids(b []byte) ([]cid.Cid, error) { + br := bytes.NewReader(b) + var out []cid.Cid + for { + _, c, err := cid.CidFromReader(br) + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + out = append(out, c) + } + + return out, nil } type userView struct { @@ -720,15 +756,15 @@ func (cs *CarStore) putShard(ctx context.Context, shard *CarShard, brefs []map[s } if len(rmcids) > 0 { - var torm []staleRef + cids := make([]cid.Cid, 0, len(rmcids)) for c := range rmcids { - torm = append(torm, staleRef{ - Cid: models.DbCID{c}, - Usr: shard.Usr, - }) + cids = append(cids, c) } - if err := tx.Create(torm).Error; err != nil { + if err := tx.Create(&staleRef{ + Cids: packCids(cids), + Usr: shard.Usr, + }).Error; err != nil { return err } } @@ -993,7 +1029,7 @@ func (cs *CarStore) WipeUserData(ctx context.Context, user models.Uid) error { } func (cs *CarStore) deleteShards(ctx context.Context, shs []*CarShard) error { - ctx, span := otel.Tracer("carstore").Start(ctx, "deleteShard") + ctx, span := otel.Tracer("carstore").Start(ctx, "deleteShards") defer span.End() deleteSlice := func(ctx context.Context, subs []*CarShard) error { @@ -1213,7 +1249,6 @@ type CompactionStats struct { NewShards int `json:"newShards"` SkippedShards int `json:"skippedShards"` ShardsDeleted int `json:"shardsDeleted"` - RefsDeleted int `json:"refsDeleted"` DupeCount int `json:"dupeCount"` } @@ -1256,7 +1291,13 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co stale := make(map[cid.Cid]bool) for _, br := range staleRefs { - stale[br.Cid.CID] = true + cids, err := br.getCids() + if err != nil { + return nil, fmt.Errorf("failed to unpack cids from staleRefs record (%d): %w", br.ID, err) + } + for _, c := range cids { + stale[c] = true + } } // if we have a staleRef that references multiple blockRefs, we consider that block a 'dirty duplicate' @@ -1389,20 +1430,17 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid) (*Co } // now we need to delete the staleRefs we successfully cleaned up - // we can delete a staleRef if all the shards that have blockRefs with matching stale refs were processed - - num, err := cs.deleteStaleRefs(ctx, brefs, staleRefs, removedShards) - if err != nil { + // we can safely delete a staleRef if all the shards that have blockRefs with matching stale refs were processed + if err := cs.deleteStaleRefs(ctx, user, brefs, staleRefs, removedShards); err != nil { return nil, err } - stats.RefsDeleted = num stats.DupeCount = len(dupes) return stats, nil } -func (cs *CarStore) deleteStaleRefs(ctx context.Context, brefs []blockRef, staleRefs []staleRef, removedShards map[uint]bool) (int, error) { +func (cs *CarStore) deleteStaleRefs(ctx context.Context, uid models.Uid, brefs []blockRef, staleRefs []staleRef, removedShards map[uint]bool) error { ctx, span := otel.Tracer("carstore").Start(ctx, "deleteStaleRefs") defer span.End() @@ -1411,35 +1449,42 @@ func (cs *CarStore) deleteStaleRefs(ctx context.Context, brefs []blockRef, stale brByCid[br.Cid.CID] = append(brByCid[br.Cid.CID], br) } - var staleToDelete []uint + var staleToKeep []cid.Cid for _, sr := range staleRefs { - brs := brByCid[sr.Cid.CID] - del := true - for _, br := range brs { - if !removedShards[br.Shard] { - del = false - break - } + cids, err := sr.getCids() + if err != nil { + return fmt.Errorf("getCids on staleRef failed (%d): %w", sr.ID, err) } - if del { - staleToDelete = append(staleToDelete, sr.ID) + for _, c := range cids { + brs := brByCid[c] + del := true + for _, br := range brs { + if !removedShards[br.Shard] { + del = false + break + } + } + + if !del { + staleToKeep = append(staleToKeep, c) + } } } - chunkSize := 10000 - for i := 0; i < len(staleToDelete); i += chunkSize { - sl := staleToDelete[i:] - if len(sl) > chunkSize { - sl = sl[:chunkSize] - } + if err := cs.meta.Delete(&staleRef{}, "usr = ?", uid).Error; err != nil { + return err + } - if err := cs.meta.Delete(&staleRef{}, "id in (?)", sl).Error; err != nil { - return 0, err - } + // now create a new staleRef with all the refs we couldnt clear out + if err := cs.meta.Create(&staleRef{ + Usr: uid, + Cids: packCids(staleToKeep), + }).Error; err != nil { + return err } - return len(staleToDelete), nil + return nil } func (cs *CarStore) compactBucket(ctx context.Context, user models.Uid, b *compBucket, shardsById map[uint]CarShard, keep map[cid.Cid]bool) error { -- 2.51.2 From 1551980f5e143d69a0b70cec5dc824786629f00d Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 6 Oct 2023 14:51:54 -0700 Subject: [PATCH 129/192] update READMEs and dev docs --- HACKING.md | 67 ++++++++++++++------------------------------- README.md | 48 ++++++++++++++++++++------------ cmd/beemo/README.md | 17 ++++++++++++ 3 files changed, 69 insertions(+), 63 deletions(-) create mode 100644 cmd/beemo/README.md diff --git a/HACKING.md b/HACKING.md index 4c90b9ab..f36821aa 100644 --- a/HACKING.md +++ b/HACKING.md @@ -1,23 +1,28 @@ ## git repo contents -Commands (run with, eg, `go run ./cmd/bigsky`): +Run with, eg, `go run ./cmd/bigsky`): - `cmd/bigsky`: BGS+indexer daemon - `cmd/palomar`: search indexer and query servcie (OpenSearch) - `cmd/gosky`: client CLI for talking to a PDS - `cmd/lexgen`: codegen tool for lexicons (Lexicon JSON to Go package) -- `cmd/laputa`: PDS daemon +- `cmd/laputa`: partial PDS daemon (not usable or under development) - `cmd/stress`: connects to local/default PDS and creates a ton of random posts - `cmd/beemo`: slack bot for moderation reporting (Bluesky Moderation Observer) - `cmd/fakermaker`: helper to generate fake accounts and content for testing +- `cmd/supercollider`: event stream load generation tool +- `cmd/sonar`: event stream monitoring tool - `gen`: dev tool to run CBOR type codegen Packages: - `api`: mostly output of lexgen (codegen) for lexicons: structs, CBOR marshaling. some higher-level code, and a PLC client (may rename) - - `api/atprot`: generated types for `com.atproto` lexicon + - `api/atproto`: generated types for `com.atproto` lexicon - `api/bsky`: generated types for `app.bsky` lexicon +- `atproto/crypto`: crytographic helpers (signing, key generation and serialization) +- `atproto/syntax`: string types and parsers for identifiers, datetimes, etc +- `atproto/identity`: DID and handle resolution - `bgs`: server implementation for crawling, etc - `carstore`: library for storing repo data in CAR files on disk, plus a metadata SQL db - `events`: types, codegen CBOR helpers, and persistence for event feeds @@ -35,26 +40,22 @@ Packages: - `util`: a few common definitions (may rename) - `xrpc`: XRPC client (not server) helpers -Other: -- `testscripts`: shell scripts that run gosky (CLI client) against local PDS - - -## jargon +## Jargon - BGS: Big Graph Service (or Server), which centrals crawls/consumes content from "all" PDSs and re-broadcasts as a firehose - PDS: Personal Data Server (or Service), which stores user atproto repositories and acts as a user agent in the network - CLI: Command Line Tool - CBOR: a binary serialization format, smilar to JSON -- PLC: "placeholder" DID provider -- DID: Decentralized IDentifier, a flexible W3C specification for persistent identifiers in URI form (eg, "did:plc:abcd1234") +- PLC: "placeholder" DID provider, see +- DID: Decentralized IDentifier, a flexible W3C specification for persistent identifiers in URI form (eg, `did:plc:abcd1234`) - XRPC: atproto convention for HTTP GET and POST endpoints specified by namespaced Lexicon schemas - CAR: simple file format for storing binary content-addressed blocks/blobs, sort of like .tar files - CID: content identifier for binary blobs, basically a flexible encoding of hash values - MST: Merkle Search Tree, a key/value map data structure using content addressed nodes -## lexicon and CBOR marshaling code generation +## Lexicon and CBOR code generation `gen/main.go` has a list of types internal to packages in this repo which need CBOR helper codegen. If you edit those types, or update the listed types/packages, re-run codegen like: @@ -64,26 +65,22 @@ Other: # then generate go run ./gen -To run codegen for new or updated Lexicons, using lexgen, first place (or git -checkout) the JSON lexicon files at `../atproto/`. -Then, in *this* repository (indigo), run commands like: +To run codegen for new or updated Lexicons, using lexgen, first place (or git checkout) the JSON lexicon files at `../atproto/`. Then, in *this* repository (indigo), run commands like: go run ./cmd/lexgen/ --package bsky --prefix app.bsky --outdir api/bsky ../atproto/lexicons/app/bsky/ go run ./cmd/lexgen/ --package atproto --prefix com.atproto --outdir api/atproto ../atproto/lexicons/com/atproto/ You may want to delete all the codegen files before re-generating, to detect deleted files. -It can require some manual munging between the lexgen step and a later `go run ./gen` to make sure things compile at least temporarily; otherwise the `gen` will not run. -In some cases, you might also need to add new types to ./gen/main.go. +It can require some manual munging between the lexgen step and a later `go run ./gen` to make sure things compile at least temporarily; otherwise the `gen` will not run. In some cases, you might also need to add new types to `./gen/main.go`. -To generate server stubs and handlers, push them in a temporary directory -first, then merge changes in to the actual PDS code: +To generate server stubs and handlers, push them in a temporary directory first, then merge changes in to the actual PDS code: mkdir tmppds go run ./cmd/lexgen/ --package pds --gen-server --types-import com.atproto:github.com/bluesky-social/indigo/api/atproto --types-import app.bsky:github.com/bluesky-social/indigo/api/bsky --outdir tmppds --gen-handlers ../atproto/lexicons -## tips and tricks +## Tips and Tricks When debugging websocket streams, the `websocat` tool (rust) can be helpful. CBOR binary is sort of mangled in to text by default. Eg: @@ -103,43 +100,21 @@ Set the log level to be more verbose, using an env variable: GOLOG_LOG_LEVEL=info go run ./cmd/pds -## gosky basic usage +## `gosky` basic usage Running against local typescript PDS in `dev-env` mode: # as "alice" user go run ./cmd/gosky/ --pds http://localhost:2583 createSession alice.test hunter2 > bsky.auth -The `bsky.auth` file is the default place that `gosky` and other client -commands will look for auth info. - - -## slack report bot basic usage - -You need an admin token, slack webhook URL, and auth file (see gosky above). -The auth file isn't actually used, only the admin token. - - # configure a slack webhook - export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T028K87/B04NBDB/oWbsHasdf23r2d - - # example pulling admin token out of `pass` password manager - export ATP_AUTH_ADMIN_PASSWORD=`pass bsky/pds-admin-staging | head -n1` - - # example just setting admin token directly - export ATP_AUTH_ADMIN_PASSWORD="someinsecurething123" +The `bsky.auth` file is the default place that `gosky` and other client commands will look for auth info. - # run the bot - GOLOG_LOG_LEVEL=debug go run ./cmd/beemo/ --pds https://pds.staging.example.com --auth bsky.auth notify-reports -## integrated development +## Integrated Development -Sometimes it is helpful to run a PLC, PDS, BGS, labelmaker, and other -components, all locally on your laptop, across languages. This section -describes one setup for this. +Sometimes it is helpful to run a PLC, PDS, BGS, labelmaker, and other components, all locally on your laptop, across languages. This section describes one setup for this. -First, you need PostgreSQL running locally. This could be via docker, or the -following commands assume some kind of debian/ubuntu setup with a postgres -server package installed and running. +First, you need PostgreSQL running locally. This could be via docker, or the following commands assume some kind of debian/ubuntu setup with a postgres server package installed and running. Create a user and databases for PLC+PDS: diff --git a/README.md b/README.md index 72e7417e..2d72b755 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,38 @@ ![photo](https://static.bnewbold.net/tmp/indigo_serac.jpeg) -indigo: golang code for Bluesky's atproto services -================================================== +indigo: atproto libraries and services in golang +================================================ Some Bluesky software is developed in Typescript, and lives in the [bluesky-social/atproto](https://github.com/bluesky-social/atproto) repository. Some is developed in Go, and lives here. -

+Services implemented in this repository: + +* **`bigsky`** ([README](./cmd/bigsky/README.md)): "Big Graph Service" (BGS) reference implementation, running at `bsky.network` +* **`palomar`** ([README](./cmd/palomar/README.md)): fulltext search service for -Everything in this repository is an work in progress. Features and "Lexicons" may be removed or updated, software interfaces broken, etc. -We are developing in the open, but not ready to accept or review significant contributions. Keep checking back! +## Development Quickstart + +All the packages in this repository are under active development. Features and software interfaces have not stabilized and may break or be removed.

+First, you will need the Go toolchain installed. We develop using the latest stable version of the language. + +The Makefile provides wrapper commands for basic development: + + make build + make test + make fmt + make lint + +Individual commands can be run like: + + go run ./cmd/bigsky + +The [HACKING](./HACKING.md) file has a list of commands and packages in this repository and some other development tips. + ## What is atproto? @@ -21,25 +40,20 @@ We are developing in the open, but not ready to accept or review significant con The Authenticated Transfer Protocol ("ATP" or "atproto") is a decentralized social media protocol, developed by [Bluesky PBC](https://blueskyweb.xyz). Learn more at: -- [Protocol Documentation](https://atproto.com/docs) -- [Overview Guide](https://atproto.com/guides/overview) 👈 Good place to start +- [Overview and Guides](https://atproto.com/guides/overview) 👈 Best starting point +- [Github Discussions](https://github.com/bluesky-social/atproto/discussions) 👈 Great place to ask questions +- [Protocol Specifications](https://atproto.com/specs/atp) - [Blogpost on self-authenticating data structures](https://blueskyweb.xyz/blog/3-6-2022-a-self-authenticating-social-protocol) +The Bluesky Social application encompasses a set of schemas and APIs built in the overall AT Protocol framework. The namespace for these "Lexicons" is `app.bsky.*`. -## Development -First, you will need the Go toolchain installed. We develop using the latest stable version of the language. +## Contributions -The Makefile provides wrapper commands for basic development: +We are working in the open, but not ready to actively collaborate on larger contributions to this codebase. - make build - make test - make fmt - make lint +Please at least open an issue ahead of time, *before* starting any non-trivial work that you hope to get reviewed or merged to this repo. -Individual commands can be run like: - - go run ./cmd/bigsky ## Are you a developer interested in building on atproto? diff --git a/cmd/beemo/README.md b/cmd/beemo/README.md new file mode 100644 index 00000000..4908395c --- /dev/null +++ b/cmd/beemo/README.md @@ -0,0 +1,17 @@ + +## beemo: Slack notification bot for moderation reports + +You need an admin token, slack webhook URL, and auth file (see gosky docs). +The auth file isn't actually used, only the admin token. + + # configure a slack webhook + export SLACK_WEBHOOK_URL=https://hooks.slack.com/services/T028K87/B04NBDB/oWbsHasdf23r2d + + # example pulling admin token out of `pass` password manager + export ATP_AUTH_ADMIN_PASSWORD=`pass bsky/pds-admin-staging | head -n1` + + # example just setting admin token directly + export ATP_AUTH_ADMIN_PASSWORD="someinsecurething123" + + # run the bot + GOLOG_LOG_LEVEL=debug go run ./cmd/beemo/ --pds https://pds.staging.example.com --auth bsky.auth notify-reports -- 2.51.2 From 117c84474eb124671ec8e19ec65b35be585e49d8 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Fri, 6 Oct 2023 18:01:40 -0700 Subject: [PATCH 130/192] README: more updates/structure --- README.md | 55 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 2d72b755..0637586b 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,32 @@ indigo: atproto libraries and services in golang Some Bluesky software is developed in Typescript, and lives in the [bluesky-social/atproto](https://github.com/bluesky-social/atproto) repository. Some is developed in Go, and lives here. -Services implemented in this repository: +## What is in here? -* **`bigsky`** ([README](./cmd/bigsky/README.md)): "Big Graph Service" (BGS) reference implementation, running at `bsky.network` -* **`palomar`** ([README](./cmd/palomar/README.md)): fulltext search service for +**Go Services:** +* **bigsky** ([README](./cmd/bigsky/README.md)): "Big Graph Service" (BGS) reference implementation, running at `bsky.network` +* **palomar** ([README](./cmd/palomar/README.md)): fulltext search service for -## Development Quickstart +**Go Packages:** + +> ⚠️ All the packages in this repository are under active development. Features and software interfaces have not stabilized and may break or be removed. + +| Package | Docs | +| -------- | ---- | +| `api/atproto`: generated types for `com.atproto.*` Lexicons | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/api/atproto)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/api/atproto) | +| `api/bsky`: generated types for `app.bsky.*` Lexicons | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/api/bsky)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/api/bsky) | +| `atproto/crypto`: crytographic signing and key serialization | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/crypto)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/crypto) | +| `atproto/identity`: DID and handle resolution | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/identity)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/identity) | +| `atproto/syntax`: string types and parsers for identifiers | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/atproto/syntax)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/atproto/syntax) | +| `mst`: Merkle Search Tree implementation | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/mst)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/mst) | +| `repo`: account data storage | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/repo)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/repo) | +| `xrpc`: HTTP API client | [![PkgGoDev](https://pkg.go.dev/badge/mod/github.com/bluesky-social/indigo/xrpc)](https://pkg.go.dev/mod/github.com/bluesky-social/indigo/xrpc) | -All the packages in this repository are under active development. Features and software interfaces have not stabilized and may break or be removed. +The TypeScript reference implementation, including PDS and bsky AppView services, is at [bluesky-social/atproto](https://github.com/bluesky-social/atproto). Source code for the Bluesky Social client app (for web and mobile) can be found at [bluesky-social/social-app](https://github.com/bluesky-social/social-app). -

+ +## Development Quickstart First, you will need the Go toolchain installed. We develop using the latest stable version of the language. @@ -50,9 +65,29 @@ The Bluesky Social application encompasses a set of schemas and APIs built in th ## Contributions -We are working in the open, but not ready to actively collaborate on larger contributions to this codebase. +> While we do accept contributions, we prioritize high quality issues and pull requests. Adhering to the below guidelines will ensure a more timely review. + +**Rules:** -Please at least open an issue ahead of time, *before* starting any non-trivial work that you hope to get reviewed or merged to this repo. +- We may not respond to your issue or PR. +- We may close an issue or PR without much feedback. +- We may lock discussions or contributions if our attention is getting DDOSed. +- We do not provide support for build issues. + +**Guidelines:** + +- Check for existing issues before filing a new one, please. +- Open an issue and give some time for discussion before submitting a PR. +- Issues are for bugs & feature requests related to the golang implementation of atproto and related services. + - For high-level discussions, please use the [Discussion Forum](https://github.com/bluesky-social/atproto/discussions). + - For client issues, please use the relevant [social-app](https://github.com/bluesky-social/social-app) repo. +- Stay away from PRs that: + - Refactor large parts of the codebase + - Add entirely new features without prior discussion + - Change the tooling or libraries used without prior discussion + - Introduce new unnecessary dependencies + +Remember, we serve a wide community of users. Our day-to-day involves us constantly asking "which top priority is our top priority." If you submit well-written PRs that solve problems concisely, that's an awesome contribution. Otherwise, as much as we'd love to accept your ideas and contributions, we really don't have the bandwidth. ## Are you a developer interested in building on atproto? @@ -60,3 +95,7 @@ Please at least open an issue ahead of time, *before* starting any non-trivial w Bluesky is an open social network built on the AT Protocol, a flexible technology that will never lock developers out of the ecosystems that they help build. With atproto, third-party can be as seamless as first-party through custom feeds, federated services, clients, and more. If you're a developer interested in building on atproto, we'd love to email you a Bluesky invite code. Simply share your GitHub (or similar) profile with us via [this form](https://forms.gle/BF21oxVNZiDjDhXF9). + +## License + +MIT License. See [LICENSE.md](./LICENSE.md) for full text. -- 2.51.2 From f81c7ac544be0f7d3eef4b7f71286510141c121a Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 04:44:44 +0000 Subject: [PATCH 131/192] Add verbose logging to handle updates --- search/indexing.go | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/search/indexing.go b/search/indexing.go index cc54f037..a671cb67 100644 --- a/search/indexing.go +++ b/search/indexing.go @@ -11,6 +11,7 @@ import ( appbsky "github.com/bluesky-social/indigo/api/bsky" "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/util" "github.com/ipfs/go-cid" @@ -132,14 +133,35 @@ func (s *Server) indexProfile(ctx context.Context, ident *identity.Identity, rec return nil } -func (s *Server) updateUserHandle(ctx context.Context, did, handle string) error { - log := s.logger.With("repo", did, "op", "updateUserHandle", "handle", handle) +func (s *Server) updateUserHandle(ctx context.Context, didStr, handle string) error { + log := s.logger.With("repo", didStr, "op", "updateUserHandle", "handle_from_event", handle) + + did, err := syntax.ParseAtIdentifier(didStr) + if err != nil || did == nil { + log.Warn("failed to parse DID", "err", err) + return err + } + + err = s.dir.Purge(ctx, *did) + if err != nil { + log.Warn("failed to purge DID from directory", "err", err) + return err + } + + ident, err := s.dir.LookupDID(ctx, syntax.DID(didStr)) + if err != nil { + log.Warn("failed to lookup DID in directory", "err", err) + return err + } + + log.Info("updating user handle", "handle_from_dir", handle) + b, err := json.Marshal(map[string]any{ "script": map[string]any{ "source": "ctx._source.handle = params.handle", "lang": "painless", "params": map[string]any{ - "handle": handle, + "handle": ident.Handle, }, }, }) @@ -150,7 +172,7 @@ func (s *Server) updateUserHandle(ctx context.Context, did, handle string) error req := esapi.UpdateRequest{ Index: s.profileIndex, - DocumentID: did, + DocumentID: didStr, Body: bytes.NewReader(b), } -- 2.51.2 From a9092c7f22a2e74791468225bc3a2ec8f844168c Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 04:55:16 +0000 Subject: [PATCH 132/192] Pass a DID to handler --- search/firehose.go | 7 ++++++- search/indexing.go | 17 ++++++----------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/search/firehose.go b/search/firehose.go index 58ea390b..5b71a5ba 100644 --- a/search/firehose.go +++ b/search/firehose.go @@ -153,7 +153,12 @@ func (s *Server) RunIndexer(ctx context.Context) error { }, RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error { - if err := s.updateUserHandle(ctx, evt.Did, evt.Handle); err != nil { + did, err := syntax.ParseDID(evt.Did) + if err != nil { + s.logger.Error("bad DID in RepoHandle event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) + return nil + } + if err := s.updateUserHandle(ctx, did, evt.Handle); err != nil { // TODO: handle this case (instead of return nil) s.logger.Error("failed to update user handle", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) } diff --git a/search/indexing.go b/search/indexing.go index a671cb67..4729cb02 100644 --- a/search/indexing.go +++ b/search/indexing.go @@ -133,22 +133,17 @@ func (s *Server) indexProfile(ctx context.Context, ident *identity.Identity, rec return nil } -func (s *Server) updateUserHandle(ctx context.Context, didStr, handle string) error { - log := s.logger.With("repo", didStr, "op", "updateUserHandle", "handle_from_event", handle) +func (s *Server) updateUserHandle(ctx context.Context, did syntax.DID, handle string) error { + log := s.logger.With("repo", did.String(), "op", "updateUserHandle", "handle_from_event", handle) - did, err := syntax.ParseAtIdentifier(didStr) - if err != nil || did == nil { - log.Warn("failed to parse DID", "err", err) - return err - } - - err = s.dir.Purge(ctx, *did) + didID := syntax.AtIdentifier{Inner: did} + err := s.dir.Purge(ctx, didID) if err != nil { log.Warn("failed to purge DID from directory", "err", err) return err } - ident, err := s.dir.LookupDID(ctx, syntax.DID(didStr)) + ident, err := s.dir.LookupDID(ctx, did) if err != nil { log.Warn("failed to lookup DID in directory", "err", err) return err @@ -172,7 +167,7 @@ func (s *Server) updateUserHandle(ctx context.Context, didStr, handle string) er req := esapi.UpdateRequest{ Index: s.profileIndex, - DocumentID: didStr, + DocumentID: did.String(), Body: bytes.NewReader(b), } -- 2.51.2 From f809259c10bb507a4f2c439eb9519eb4304ecf2b Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 22:00:53 -0700 Subject: [PATCH 133/192] syntax: helpers to convert DID and Handle to AtIdentifier directly --- atproto/syntax/atidentifier_test.go | 2 +- atproto/syntax/did.go | 4 ++++ atproto/syntax/did_test.go | 3 +++ atproto/syntax/handle.go | 4 ++++ atproto/syntax/handle_test.go | 3 +++ 5 files changed, 15 insertions(+), 1 deletion(-) diff --git a/atproto/syntax/atidentifier_test.go b/atproto/syntax/atidentifier_test.go index b8bce027..0b2140ae 100644 --- a/atproto/syntax/atidentifier_test.go +++ b/atproto/syntax/atidentifier_test.go @@ -71,7 +71,7 @@ func TestDowncase(t *testing.T) { assert.NoError(err) } -func TestEmtpy(t *testing.T) { +func TestEmpty(t *testing.T) { assert := assert.New(t) atid := AtIdentifier{} diff --git a/atproto/syntax/did.go b/atproto/syntax/did.go index 2289f1f8..b7bf421e 100644 --- a/atproto/syntax/did.go +++ b/atproto/syntax/did.go @@ -46,6 +46,10 @@ func (d DID) Identifier() string { return parts[2] } +func (d DID) AtIdentifier() AtIdentifier { + return AtIdentifier{Inner: d} +} + func (d DID) String() string { return string(d) } diff --git a/atproto/syntax/did_test.go b/atproto/syntax/did_test.go index 36f7fc51..2c085bc8 100644 --- a/atproto/syntax/did_test.go +++ b/atproto/syntax/did_test.go @@ -55,6 +55,7 @@ func TestDIDParts(t *testing.T) { assert.NoError(err) assert.Equal("example", d.Method()) assert.Equal("123456789abcDEFghi", d.Identifier()) + assert.Equal(d.String(), d.AtIdentifier().String()) } func TestDIDNoPanic(t *testing.T) { @@ -62,5 +63,7 @@ func TestDIDNoPanic(t *testing.T) { bad := DID(s) _ = bad.Identifier() _ = bad.Method() + _ = bad.AtIdentifier() + _ = bad.AtIdentifier().String() } } diff --git a/atproto/syntax/handle.go b/atproto/syntax/handle.go index 0efe2917..63322b7f 100644 --- a/atproto/syntax/handle.go +++ b/atproto/syntax/handle.go @@ -62,6 +62,10 @@ func (h Handle) Normalize() Handle { return Handle(strings.ToLower(string(h))) } +func (h Handle) AtIdentifier() AtIdentifier { + return AtIdentifier{Inner: h} +} + func (h Handle) String() string { return string(h) } diff --git a/atproto/syntax/handle_test.go b/atproto/syntax/handle_test.go index 5825b545..3d591de3 100644 --- a/atproto/syntax/handle_test.go +++ b/atproto/syntax/handle_test.go @@ -57,6 +57,7 @@ func TestHandleNormalize(t *testing.T) { assert.NoError(err) assert.Equal("john.test", string(handle.Normalize())) assert.NoError(err) + assert.Equal(handle.String(), handle.AtIdentifier().String()) _, err = ParseHandle("JoH!n.TeST") assert.Error(err) @@ -68,6 +69,8 @@ func TestHandleNoPanic(t *testing.T) { _ = bad.Normalize() _ = bad.TLD() _ = bad.AllowedTLD() + _ = bad.AtIdentifier() + _ = bad.AtIdentifier().String() } } func TestHandleInterfaces(t *testing.T) { -- 2.51.2 From 314547482979917e9054d5bf07c99445084e5dfc Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 05:13:56 +0000 Subject: [PATCH 134/192] Use new AtIdentifier helper --- search/indexing.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/search/indexing.go b/search/indexing.go index 4729cb02..042961df 100644 --- a/search/indexing.go +++ b/search/indexing.go @@ -136,8 +136,7 @@ func (s *Server) indexProfile(ctx context.Context, ident *identity.Identity, rec func (s *Server) updateUserHandle(ctx context.Context, did syntax.DID, handle string) error { log := s.logger.With("repo", did.String(), "op", "updateUserHandle", "handle_from_event", handle) - didID := syntax.AtIdentifier{Inner: did} - err := s.dir.Purge(ctx, didID) + err := s.dir.Purge(ctx, did.AtIdentifier()) if err != nil { log.Warn("failed to purge DID from directory", "err", err) return err -- 2.51.2 From a38a41d83b5ffd8248503da6541873d16a7838c6 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 21:21:44 -0700 Subject: [PATCH 135/192] make fmt --- api/bsky/embedimages.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/bsky/embedimages.go b/api/bsky/embedimages.go index b0cdb8a6..927f3ff7 100644 --- a/api/bsky/embedimages.go +++ b/api/bsky/embedimages.go @@ -19,8 +19,8 @@ type EmbedImages struct { // EmbedImages_Image is a "image" in the app.bsky.embed.images schema. type EmbedImages_Image struct { - Alt string `json:"alt" cborgen:"alt"` - Image *util.LexBlob `json:"image" cborgen:"image"` + Alt string `json:"alt" cborgen:"alt"` + Image *util.LexBlob `json:"image" cborgen:"image"` } // EmbedImages_View is a "view" in the app.bsky.embed.images schema. @@ -33,7 +33,7 @@ type EmbedImages_View struct { // EmbedImages_ViewImage is a "viewImage" in the app.bsky.embed.images schema. type EmbedImages_ViewImage struct { - Alt string `json:"alt" cborgen:"alt"` - Fullsize string `json:"fullsize" cborgen:"fullsize"` - Thumb string `json:"thumb" cborgen:"thumb"` + Alt string `json:"alt" cborgen:"alt"` + Fullsize string `json:"fullsize" cborgen:"fullsize"` + Thumb string `json:"thumb" cborgen:"thumb"` } -- 2.51.2 From 29748f9a127eb61c6aa999283352543727a50397 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 16:44:08 +0000 Subject: [PATCH 136/192] Fix nil identity null pointer --- search/indexing.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/search/indexing.go b/search/indexing.go index 042961df..da221a42 100644 --- a/search/indexing.go +++ b/search/indexing.go @@ -148,6 +148,11 @@ func (s *Server) updateUserHandle(ctx context.Context, did syntax.DID, handle st return err } + if ident == nil { + log.Warn("got nil identity from directory") + return fmt.Errorf("got nil identity from directory") + } + log.Info("updating user handle", "handle_from_dir", handle) b, err := json.Marshal(map[string]any{ -- 2.51.2 From d8d59708975c8a64117d68f1c96c14b5490c3482 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 17:18:11 +0000 Subject: [PATCH 137/192] Fix log message --- search/indexing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/indexing.go b/search/indexing.go index da221a42..f4851cd6 100644 --- a/search/indexing.go +++ b/search/indexing.go @@ -153,7 +153,7 @@ func (s *Server) updateUserHandle(ctx context.Context, did syntax.DID, handle st return fmt.Errorf("got nil identity from directory") } - log.Info("updating user handle", "handle_from_dir", handle) + log.Info("updating user handle", "handle_from_dir", ident.Handle) b, err := json.Marshal(map[string]any{ "script": map[string]any{ -- 2.51.2 From 6fad7c2004ef388e382fd1116bf1e5b8d51c37d9 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 10 Oct 2023 11:42:03 -0700 Subject: [PATCH 138/192] address review feedback --- carstore/bs.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/carstore/bs.go b/carstore/bs.go index 06ea9e02..893b8a2e 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1472,16 +1472,24 @@ func (cs *CarStore) deleteStaleRefs(ctx context.Context, uid models.Uid, brefs [ } } - if err := cs.meta.Delete(&staleRef{}, "usr = ?", uid).Error; err != nil { + txn := cs.meta.Begin() + + if err := txn.Delete(&staleRef{}, "usr = ?", uid).Error; err != nil { return err } // now create a new staleRef with all the refs we couldnt clear out - if err := cs.meta.Create(&staleRef{ - Usr: uid, - Cids: packCids(staleToKeep), - }).Error; err != nil { - return err + if len(staleToKeep) > 0 { + if err := txn.Create(&staleRef{ + Usr: uid, + Cids: packCids(staleToKeep), + }).Error; err != nil { + return err + } + } + + if err := txn.Commit().Error; err != nil { + return fmt.Errorf("failed to commit staleRef updates: %w", err) } return nil -- 2.51.2 From d08033e7c7297287057e61a4c2c18cfd8b6dd85e Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 20:18:39 -0700 Subject: [PATCH 139/192] palomar: remove unused 'elastic-cert' flag from 'elastic-check' cmd --- cmd/palomar/main.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/palomar/main.go b/cmd/palomar/main.go index f7a6887e..1938d52d 100644 --- a/cmd/palomar/main.go +++ b/cmd/palomar/main.go @@ -219,12 +219,8 @@ var runCmd = &cli.Command{ } var elasticCheckCmd = &cli.Command{ - Name: "elastic-check", - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "elastic-cert", - }, - }, + Name: "elastic-check", + Flags: []cli.Flag{}, Action: func(cctx *cli.Context) error { escli, err := createEsClient(cctx) if err != nil { -- 2.51.2 From 4684c52f7cd3ff37a6eb791de9e61dc63eaada5f Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 20:19:27 -0700 Subject: [PATCH 140/192] palomar: CLI arg to skip SSL cert validation (for dev) --- cmd/palomar/main.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cmd/palomar/main.go b/cmd/palomar/main.go index 1938d52d..2743afee 100644 --- a/cmd/palomar/main.go +++ b/cmd/palomar/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/tls" "encoding/json" "fmt" "log/slog" @@ -43,6 +44,11 @@ func run(args []string) error { Usage: "certificate file path", EnvVars: []string{"ES_CERT_FILE", "ELASTIC_CERT_FILE"}, }, + &cli.BoolFlag{ + Name: "elastic-insecure-ssl", + Usage: "if true, disable SSL cert validation", + EnvVars: []string{"ES_INSECURE_SSL"}, + }, &cli.StringFlag{ Name: "elastic-username", Usage: "elasticsearch username", @@ -349,6 +355,8 @@ func createEsClient(cctx *cli.Context) (*es.Client, error) { cert = b } + insecure := cctx.Bool("elastic-insecure-ssl") + cfg := es.Config{ Addresses: addrs, Username: cctx.String("elastic-username"), @@ -356,6 +364,9 @@ func createEsClient(cctx *cli.Context) (*es.Client, error) { CACert: cert, Transport: &http.Transport{ MaxIdleConnsPerHost: 20, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: insecure, + }, }, } -- 2.51.2 From 63b816a0f9708899918fa5e1c25c632605f05671 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 20:20:47 -0700 Subject: [PATCH 141/192] palomar: improve profile typeahead search quality - 'and' operation on terms (not 'or') - special-case '@'-prefixed query as exact handle prefix match - special-case single-word query to have some boosts on handle prefix match, and custom domain over bsky.social --- search/query.go | 52 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/search/query.go b/search/query.go index d58ca9ad..c4aad5a1 100644 --- a/search/query.go +++ b/search/query.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "log/slog" + "strings" "github.com/bluesky-social/indigo/atproto/identity" @@ -126,12 +127,15 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s if err := checkParams(0, size); err != nil { return nil, err } + query := map[string]interface{}{ "query": map[string]interface{}{ "multi_match": map[string]interface{}{ - "query": q, - "type": "bool_prefix", + "query": q, + "type": "bool_prefix", + "operator": "and", "fields": []string{ + "handle^2", "typeahead", "typeahead._2gram", "typeahead._3gram", @@ -141,6 +145,50 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s "size": size, } + // special-case: exact string match of handle + if strings.HasPrefix(q, "@") && !strings.Contains(q, " ") { + q = q[1:] + query["query"] = map[string]interface{}{ + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q, + }, + }, + } + } + + // boost exact handle prefix match, if q is single simple term + if len(q) >= 3 && !strings.ContainsAny(q, " .") { + query["rescore"] = map[string]interface{}{ + "window_size": 100, + "query": map[string]interface{}{ + "rescore_query": map[string]interface{}{ + "boosting": map[string]interface{}{ + "positive": map[string]interface{}{ + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q + ".", + }, + }, + }, + // downrank *.bsky.social (vs custom domain) + // wildcard is expensive, so only in rescore + "negative": map[string]interface{}{ + "wildcard": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": "*.bsky.social", + }, + }, + }, + "negative_boost": 0.5, + }, + }, + "query_weight": 1.0, + "rescore_query_weight": 1.0, + }, + } + } + return doSearch(ctx, escli, index, query) } -- 2.51.2 From 82532a2e0f5f6148315bfcf2eae36a6627dc7913 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 20:25:07 -0700 Subject: [PATCH 142/192] palomar: log opensearch error JSON --- search/query.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/search/query.go b/search/query.go index c4aad5a1..0854fa3e 100644 --- a/search/query.go +++ b/search/query.go @@ -228,7 +228,10 @@ func doSearch(ctx context.Context, escli *es.Client, index string, query interfa } defer res.Body.Close() if res.IsError() { - ioutil.ReadAll(res.Body) + raw, err := ioutil.ReadAll(res.Body) + if nil == err { + slog.Warn("search query error", "resp", string(raw), "status_code", res.StatusCode) + } return nil, fmt.Errorf("search query error, code=%d", res.StatusCode) } -- 2.51.2 From a7f7b6b73d59b61fad32c71ad20b3407a7fb7b9f Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 9 Oct 2023 21:20:13 -0700 Subject: [PATCH 143/192] palomar: tweak non-typeahead profile search --- search/query.go | 59 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/search/query.go b/search/query.go index 0854fa3e..04f3a23c 100644 --- a/search/query.go +++ b/search/query.go @@ -93,17 +93,23 @@ func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Cli if err := checkParams(offset, size); err != nil { return nil, err } + queryStr, filters := ParseQuery(ctx, dir, q) basic := map[string]interface{}{ "simple_query_string": map[string]interface{}{ - "query": queryStr, - "fields": []string{"everything"}, + "query": queryStr, + "fields": []string{ + "handle^3", + "display_name^2", + "everything", + }, "flags": "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE", "default_operator": "and", "lenient": true, "analyze_wildcard": false, }, } + query := map[string]interface{}{ "query": map[string]interface{}{ "bool": map[string]interface{}{ @@ -112,14 +118,59 @@ func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Cli map[string]interface{}{"term": map[string]interface{}{"has_avatar": true}}, map[string]interface{}{"term": map[string]interface{}{"has_banner": true}}, }, - "filter": filters, - "boost": 1.0, + "minimum_should_match": 0, + "filter": filters, + "boost": 0.5, }, }, "size": size, "from": offset, } + // special-case: exact string match of handle + if strings.HasPrefix(q, "@") && !strings.Contains(q, " ") { + q = q[1:] + query["query"] = map[string]interface{}{ + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q, + }, + }, + } + } + + // boost exact handle prefix match, if q is single simple term + if len(q) >= 3 && !strings.ContainsAny(q, " .") { + query["rescore"] = map[string]interface{}{ + "window_size": 100, + "query": map[string]interface{}{ + "rescore_query": map[string]interface{}{ + "boosting": map[string]interface{}{ + "positive": map[string]interface{}{ + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q + ".", + }, + }, + }, + // downrank *.bsky.social (vs custom domain) + // wildcard is expensive, so only in rescore + "negative": map[string]interface{}{ + "wildcard": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": "*.bsky.social", + }, + }, + }, + "negative_boost": 0.5, + }, + }, + "query_weight": 0.5, + "rescore_query_weight": 2.0, + }, + } + } + return doSearch(ctx, escli, index, query) } -- 2.51.2 From c125b13f72227286629da4fa9579bf8289593af3 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 19:31:25 +0000 Subject: [PATCH 144/192] Track whether or not a query is typeahead in metrics --- search/metrics.go | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/search/metrics.go b/search/metrics.go index bd53581e..dba2aae5 100644 --- a/search/metrics.go +++ b/search/metrics.go @@ -60,24 +60,24 @@ var reqSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "http_request_size_bytes", Help: "A histogram of request sizes for requests.", Buckets: prometheus.ExponentialBuckets(100, 10, 8), -}, []string{"code", "method", "path"}) +}, []string{"code", "method", "path", "extras"}) var reqDur = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "http_request_duration_seconds", Help: "A histogram of latencies for requests.", Buckets: prometheus.ExponentialBuckets(0.0001, 2, 18), -}, []string{"code", "method", "path"}) +}, []string{"code", "method", "path", "extras"}) var reqCnt = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "http_requests_total", Help: "A counter for requests to the wrapped handler.", -}, []string{"code", "method", "path"}) +}, []string{"code", "method", "path", "extras"}) var resSz = promauto.NewHistogramVec(prometheus.HistogramOpts{ Name: "http_response_size_bytes", Help: "A histogram of response sizes for requests.", Buckets: prometheus.ExponentialBuckets(100, 10, 8), -}, []string{"code", "method", "path"}) +}, []string{"code", "method", "path", "extras"}) // MetricsMiddleware defines handler function for metrics middleware func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { @@ -110,10 +110,19 @@ func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { responseSize := float64(c.Response().Size) - reqDur.WithLabelValues(statusStr, method, path).Observe(elapsed) - reqCnt.WithLabelValues(statusStr, method, path).Inc() - reqSz.WithLabelValues(statusStr, method, path).Observe(float64(requestSize)) - resSz.WithLabelValues(statusStr, method, path).Observe(responseSize) + // Custom label for Typeahead search queries + isTypeahead := c.QueryParam("typeahead") == "true" + labels := []string{statusStr, method, path} + if isTypeahead { + labels = append(labels, "typeahead") + } else { + labels = append(labels, "") + } + + reqDur.WithLabelValues(labels...).Observe(elapsed) + reqCnt.WithLabelValues(labels...).Inc() + reqSz.WithLabelValues(labels...).Observe(float64(requestSize)) + resSz.WithLabelValues(labels...).Observe(responseSize) return err } -- 2.51.2 From 7a84f409469d63619b09416b98e0069894148bb4 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 19:33:23 +0000 Subject: [PATCH 145/192] Full typeahead options --- search/metrics.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/search/metrics.go b/search/metrics.go index dba2aae5..e76a7fa4 100644 --- a/search/metrics.go +++ b/search/metrics.go @@ -4,6 +4,7 @@ import ( "errors" "net/http" "strconv" + "strings" "time" "github.com/labstack/echo/v4" @@ -111,9 +112,13 @@ func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { responseSize := float64(c.Response().Size) // Custom label for Typeahead search queries - isTypeahead := c.QueryParam("typeahead") == "true" + typeahead := false + if q := strings.TrimSpace(c.QueryParam("typeahead")); q == "true" || q == "1" || q == "y" { + typeahead = true + } + labels := []string{statusStr, method, path} - if isTypeahead { + if typeahead { labels = append(labels, "typeahead") } else { labels = append(labels, "") -- 2.51.2 From bf9989830af251a801e4fd3c9edacf10e0657f12 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Tue, 10 Oct 2023 13:36:41 -0700 Subject: [PATCH 146/192] explain some of the choices in skipBigShards in a comment --- carstore/bs.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/carstore/bs.go b/carstore/bs.go index 1a7cd94a..efb87d93 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1244,6 +1244,15 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skip }) if skipBigShards { + // Since we generally expect shards to start bigger and get smaller, + // and because we want to avoid compacting non-adjacent shards + // together, and because we want to avoid running a stat on every + // single shard (can be expensive for repos that havent been compacted + // in a while) we only skip a prefix of shard files that are over the + // threshold. this may end up not skipping some shards that are over + // the threshold if a below-threshold shard occurs before them, but + // since this is a heuristic and imperfect optimization, that is + // acceptable. var skip int for i, sh := range shards { size, err := shardSize(&sh) -- 2.51.2 From 28305bd7216bd6f9586abe43cd3c1049048934c9 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Tue, 10 Oct 2023 20:52:18 +0000 Subject: [PATCH 147/192] Use a sentinel value rather than empty string for empty extras --- search/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/metrics.go b/search/metrics.go index e76a7fa4..9378af3b 100644 --- a/search/metrics.go +++ b/search/metrics.go @@ -121,7 +121,7 @@ func MetricsMiddleware(next echo.HandlerFunc) echo.HandlerFunc { if typeahead { labels = append(labels, "typeahead") } else { - labels = append(labels, "") + labels = append(labels, "_none") } reqDur.WithLabelValues(labels...).Observe(elapsed) -- 2.51.2 From a7fb6896ebf7654591f542dc4f6627a525610fea Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 02:03:10 +0000 Subject: [PATCH 148/192] Starting to build out compactor daemon --- bgs/bgs.go | 61 ---------- bgs/compact.go | 299 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 61 deletions(-) create mode 100644 bgs/compact.go diff --git a/bgs/bgs.go b/bgs/bgs.go index 30eee5df..c8b086ed 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -1202,67 +1202,6 @@ func (bgs *BGS) ReverseTakedown(ctx context.Context, did string) error { return nil } -type compactionStats struct { - Completed map[models.Uid]*carstore.CompactionStats - Targets []carstore.CompactionTarget -} - -func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool, fast bool) (*compactionStats, error) { - ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction") - defer span.End() - - log.Warn("starting repo compaction") - - runStart := time.Now() - - repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50) - if err != nil { - return nil, fmt.Errorf("failed to get repos to compact: %w", err) - } - - if lim > 0 && len(repos) > lim { - repos = repos[:lim] - } - - if dry { - return &compactionStats{ - Targets: repos, - }, nil - } - - results := make(map[models.Uid]*carstore.CompactionStats) - for i, r := range repos { - select { - case <-ctx.Done(): - return &compactionStats{ - Targets: repos, - Completed: results, - }, nil - default: - } - - repostart := time.Now() - st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr, fast) - if err != nil { - log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) - continue - } - compactionDuration.Observe(time.Since(repostart).Seconds()) - results[r.Usr] = st - - if i%100 == 0 { - log.Warnf("compacted %d repos in %s", i+1, time.Since(runStart)) - } - } - - log.Warnf("compacted %d repos in %s", len(repos), time.Since(runStart)) - - return &compactionStats{ - Targets: repos, - Completed: results, - }, nil -} - type revCheckResult struct { ai *models.ActorInfo err error diff --git a/bgs/compact.go b/bgs/compact.go new file mode 100644 index 00000000..e78f0ca5 --- /dev/null +++ b/bgs/compact.go @@ -0,0 +1,299 @@ +package bgs + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/bluesky-social/indigo/carstore" + "github.com/bluesky-social/indigo/models" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" +) + +type queue struct { + q []models.Uid + members map[models.Uid]struct{} + lk sync.Mutex +} + +func (q *queue) Append(uid models.Uid) { + q.lk.Lock() + defer q.lk.Unlock() + + if _, ok := q.members[uid]; ok { + return + } + + q.q = append(q.q, uid) + q.members[uid] = struct{}{} +} + +func (q *queue) Prepend(uid models.Uid) { + q.lk.Lock() + defer q.lk.Unlock() + + if _, ok := q.members[uid]; ok { + return + } + + q.q = append([]models.Uid{uid}, q.q...) + q.members[uid] = struct{}{} +} + +func (q *queue) Has(uid models.Uid) bool { + q.lk.Lock() + defer q.lk.Unlock() + + _, ok := q.members[uid] + return ok +} + +func (q *queue) Remove(uid models.Uid) { + q.lk.Lock() + defer q.lk.Unlock() + + if _, ok := q.members[uid]; !ok { + return + } + + for i, u := range q.q { + if u == uid { + q.q = append(q.q[:i], q.q[i+1:]...) + break + } + } + + delete(q.members, uid) +} + +func (q *queue) Pop() (models.Uid, bool) { + q.lk.Lock() + defer q.lk.Unlock() + + if len(q.q) == 0 { + return 0, false + } + + uid := q.q[0] + q.q = q.q[1:] + delete(q.members, uid) + + return uid, true +} + +type CompactorState struct { + latestUID models.Uid + latestDID string + status string + stats *carstore.CompactionStats +} + +type Compactor struct { + q *queue + state *CompactorState + stateLk sync.RWMutex + exit chan struct{} +} + +func NewCompactor() *Compactor { + return &Compactor{ + q: &queue{ + members: make(map[models.Uid]struct{}), + }, + } +} + +type compactionStats struct { + Completed map[models.Uid]*carstore.CompactionStats + Targets []carstore.CompactionTarget +} + +func (c *Compactor) SetState(uid models.Uid, did, status string, stats *carstore.CompactionStats) { + c.stateLk.Lock() + defer c.stateLk.Unlock() + + c.state.latestUID = uid + c.state.latestDID = did + c.state.status = status + c.state.stats = stats +} + +func (c *Compactor) GetState() *CompactorState { + c.stateLk.RLock() + defer c.stateLk.RUnlock() + + return &CompactorState{ + latestUID: c.state.latestUID, + latestDID: c.state.latestDID, + status: c.state.status, + stats: c.state.stats, + } +} + +var errNoReposToCompact = fmt.Errorf("no repos to compact") + +func (c *Compactor) Run(bgs *BGS, fast bool) { + for { + select { + case <-c.exit: + log.Warn("compactor exiting") + return + default: + } + + ctx := context.Background() + start := time.Now() + state, err := c.CompactNext(ctx, bgs, fast) + if err != nil { + if err == errNoReposToCompact { + log.Warn("no repos to compact, waiting and retrying") + time.Sleep(time.Second * 5) + continue + } + log.Errorw("failed to compact repo", + "err", err, + "uid", state.latestUID, + "repo", state.latestDID, + "status", state.status, + "stats", state.stats, + "duration", time.Since(start), + ) + // Pause for a bit to avoid spamming failed compactions + time.Sleep(time.Millisecond * 100) + } else { + log.Warnw("compacted repo", + "uid", state.latestUID, + "repo", state.latestDID, + "status", state.status, + "stats", state.stats, + "duration", time.Since(start), + ) + } + } +} + +func (c *Compactor) CompactNext(ctx context.Context, bgs *BGS, fast bool) (*CompactorState, error) { + ctx, span := otel.Tracer("bgs").Start(ctx, "CompactNext") + defer span.End() + + uid, ok := c.q.Pop() + if !ok { + return nil, errNoReposToCompact + } + + c.SetState(uid, "unknown", "getting_user", nil) + + user := User{} + if err := bgs.db.Model(&User{}).Where("id = ?", uid).First(&user).Error; err != nil { + return nil, fmt.Errorf("failed to get user: %w", err) + } + + c.SetState(uid, user.Did, "compacting", nil) + + start := time.Now() + st, err := bgs.repoman.CarStore().CompactUserShards(ctx, uid, fast) + if err != nil { + c.SetState(uid, user.Did, "failed", nil) + return nil, fmt.Errorf("failed to compact shards for user %d: %w", uid, err) + } + compactionDuration.Observe(time.Since(start).Seconds()) + + c.SetState(uid, user.Did, "done", st) + + return c.GetState(), nil +} + +func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shardCount int) error { + ctx, span := otel.Tracer("bgs").Start(ctx, "EnqueueAllRepos") + defer span.End() + + span.SetAttributes(attribute.Int("lim", lim), attribute.Int("shardCount", shardCount)) + + if shardCount == 0 { + shardCount = 50 + } + + span.SetAttributes(attribute.Int("clampedShardCount", shardCount)) + + log := log.With("source", "compactor_enqueue_all_repos") + log.Warn("enqueueing all repos") + + repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, shardCount) + if err != nil { + return fmt.Errorf("failed to get repos to compact: %w", err) + } + + span.SetAttributes(attribute.Int("repos", len(repos))) + + if lim > 0 && len(repos) > lim { + repos = repos[:lim] + } + + span.SetAttributes(attribute.Int("clampedRepos", len(repos))) + + for _, r := range repos { + c.q.Append(r.Usr) + } + + log.Warn("done enqueueing all repos") + + return nil +} + +func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool, fast bool) (*compactionStats, error) { + ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction") + defer span.End() + + log.Warn("starting repo compaction") + + runStart := time.Now() + + repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50) + if err != nil { + return nil, fmt.Errorf("failed to get repos to compact: %w", err) + } + + if lim > 0 && len(repos) > lim { + repos = repos[:lim] + } + + if dry { + return &compactionStats{ + Targets: repos, + }, nil + } + + results := make(map[models.Uid]*carstore.CompactionStats) + for i, r := range repos { + select { + case <-ctx.Done(): + return &compactionStats{ + Targets: repos, + Completed: results, + }, nil + default: + } + + repostart := time.Now() + st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr, fast) + if err != nil { + log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) + continue + } + compactionDuration.Observe(time.Since(repostart).Seconds()) + results[r.Usr] = st + + if i%100 == 0 { + log.Warnf("compacted %d repos in %s", i+1, time.Since(runStart)) + } + } + + log.Warnf("compacted %d repos in %s", len(repos), time.Since(runStart)) + + return &compactionStats{ + Targets: repos, + Completed: results, + }, nil +} -- 2.51.2 From f758618e63de56c186f822878389cc8a1f546cfa Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 18:35:21 +0000 Subject: [PATCH 149/192] More compactor work, make each compaction job indicate whether or ont it should be fast-compacted --- bgs/admin.go | 13 ++++----- bgs/bgs.go | 19 ++++++++++++ bgs/compact.go | 79 +++++++++++++++++++++++++++++++------------------- 3 files changed, 73 insertions(+), 38 deletions(-) diff --git a/bgs/admin.go b/bgs/admin.go index d83ae5bb..1e8a6d83 100644 --- a/bgs/admin.go +++ b/bgs/admin.go @@ -431,11 +431,6 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { ctx, span := otel.Tracer("bgs").Start(context.Background(), "adminCompactAllRepos") defer span.End() - var dry bool - if strings.ToLower(e.QueryParam("dry")) == "true" { - dry = true - } - var fast bool if strings.ToLower(e.QueryParam("fast")) == "true" { fast = true @@ -451,12 +446,14 @@ func (bgs *BGS) handleAdminCompactAllRepos(e echo.Context) error { lim = v } - stats, err := bgs.runRepoCompaction(ctx, lim, dry, fast) + err := bgs.compactor.EnqueueAllRepos(ctx, bgs, lim, 0, fast) if err != nil { - return fmt.Errorf("compaction run failed: %w", err) + return echo.NewHTTPError(http.StatusInternalServerError, fmt.Errorf("failed to enqueue all repos: %w", err)) } - return e.JSON(200, stats) + return e.JSON(200, map[string]any{ + "success": "true", + }) } func (bgs *BGS) handleAdminPostResyncPDS(e echo.Context) error { diff --git a/bgs/bgs.go b/bgs/bgs.go index c8b086ed..a1316e0b 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -84,6 +84,9 @@ type BGS struct { // Management of Resyncs pdsResyncsLk sync.RWMutex pdsResyncs map[uint]*PDSResync + + // Management of Compaction + compactor *Compactor } type PDSResync struct { @@ -733,6 +736,22 @@ func (bgs *BGS) lookupUserByDid(ctx context.Context, did string) (*User, error) return &u, nil } +func (bgs *BGS) lookupUserByUID(ctx context.Context, uid models.Uid) (*User, error) { + ctx, span := otel.Tracer("bgs").Start(ctx, "lookupUserByUID") + defer span.End() + + var u User + if err := bgs.db.Find(&u, "id = ?", uid).Error; err != nil { + return nil, err + } + + if u.ID == 0 { + return nil, gorm.ErrRecordNotFound + } + + return &u, nil +} + func stringLink(lnk *lexutil.LexLink) string { if lnk == nil { return "" diff --git a/bgs/compact.go b/bgs/compact.go index e78f0ca5..2feaf336 100644 --- a/bgs/compact.go +++ b/bgs/compact.go @@ -12,13 +12,18 @@ import ( "go.opentelemetry.io/otel/attribute" ) +type queueItem struct { + uid models.Uid + fast bool +} + type queue struct { - q []models.Uid + q []queueItem members map[models.Uid]struct{} lk sync.Mutex } -func (q *queue) Append(uid models.Uid) { +func (q *queue) Append(uid models.Uid, fast bool) { q.lk.Lock() defer q.lk.Unlock() @@ -26,11 +31,11 @@ func (q *queue) Append(uid models.Uid) { return } - q.q = append(q.q, uid) + q.q = append(q.q, queueItem{uid: uid, fast: fast}) q.members[uid] = struct{}{} } -func (q *queue) Prepend(uid models.Uid) { +func (q *queue) Prepend(uid models.Uid, fast bool) { q.lk.Lock() defer q.lk.Unlock() @@ -38,7 +43,7 @@ func (q *queue) Prepend(uid models.Uid) { return } - q.q = append([]models.Uid{uid}, q.q...) + q.q = append([]queueItem{{uid: uid, fast: fast}}, q.q...) q.members[uid] = struct{}{} } @@ -58,8 +63,8 @@ func (q *queue) Remove(uid models.Uid) { return } - for i, u := range q.q { - if u == uid { + for i, item := range q.q { + if item.uid == uid { q.q = append(q.q[:i], q.q[i+1:]...) break } @@ -68,19 +73,19 @@ func (q *queue) Remove(uid models.Uid) { delete(q.members, uid) } -func (q *queue) Pop() (models.Uid, bool) { +func (q *queue) Pop() (*queueItem, bool) { q.lk.Lock() defer q.lk.Unlock() if len(q.q) == 0 { - return 0, false + return nil, false } - uid := q.q[0] + item := q.q[0] q.q = q.q[1:] - delete(q.members, uid) + delete(q.members, item.uid) - return uid, true + return &item, true } type CompactorState struct { @@ -134,7 +139,7 @@ func (c *Compactor) GetState() *CompactorState { var errNoReposToCompact = fmt.Errorf("no repos to compact") -func (c *Compactor) Run(bgs *BGS, fast bool) { +func (c *Compactor) Run(bgs *BGS) { for { select { case <-c.exit: @@ -145,7 +150,7 @@ func (c *Compactor) Run(bgs *BGS, fast bool) { ctx := context.Background() start := time.Now() - state, err := c.CompactNext(ctx, bgs, fast) + state, err := c.CompactNext(ctx, bgs) if err != nil { if err == errNoReposToCompact { log.Warn("no repos to compact, waiting and retrying") @@ -174,42 +179,56 @@ func (c *Compactor) Run(bgs *BGS, fast bool) { } } -func (c *Compactor) CompactNext(ctx context.Context, bgs *BGS, fast bool) (*CompactorState, error) { +func (c *Compactor) CompactNext(ctx context.Context, bgs *BGS) (*CompactorState, error) { ctx, span := otel.Tracer("bgs").Start(ctx, "CompactNext") defer span.End() - uid, ok := c.q.Pop() - if !ok { + item, ok := c.q.Pop() + if !ok || item == nil { return nil, errNoReposToCompact } - c.SetState(uid, "unknown", "getting_user", nil) + c.SetState(item.uid, "unknown", "getting_user", nil) - user := User{} - if err := bgs.db.Model(&User{}).Where("id = ?", uid).First(&user).Error; err != nil { - return nil, fmt.Errorf("failed to get user: %w", err) + user, err := bgs.lookupUserByUID(ctx, item.uid) + if err != nil { + c.SetState(item.uid, "unknown", "failed_getting_user", nil) + return nil, fmt.Errorf("failed to get user %d: %w", item.uid, err) } - c.SetState(uid, user.Did, "compacting", nil) + c.SetState(item.uid, user.Did, "compacting", nil) start := time.Now() - st, err := bgs.repoman.CarStore().CompactUserShards(ctx, uid, fast) + st, err := bgs.repoman.CarStore().CompactUserShards(ctx, item.uid, item.fast) if err != nil { - c.SetState(uid, user.Did, "failed", nil) - return nil, fmt.Errorf("failed to compact shards for user %d: %w", uid, err) + c.SetState(item.uid, user.Did, "failed_compacting", nil) + return nil, fmt.Errorf("failed to compact shards for user %d: %w", item.uid, err) } compactionDuration.Observe(time.Since(start).Seconds()) - c.SetState(uid, user.Did, "done", st) + c.SetState(item.uid, user.Did, "done", st) return c.GetState(), nil } -func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shardCount int) error { +func (c *Compactor) EnqueueRepo(ctx context.Context, user User, fast bool) { + log.Infow("enqueueing compaction for repo", "repo", user.Did, "uid", user.ID, "fast", fast) + c.q.Append(user.ID, fast) +} + +// EnqueueAllRepos enqueues all repos for compaction +// lim is the maximum number of repos to enqueue +// shardCount is the number of shards to compact per user (0 = default of 50) +// fast is whether to use the fast compaction method (skip large shards) +func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shardCount int, fast bool) error { ctx, span := otel.Tracer("bgs").Start(ctx, "EnqueueAllRepos") defer span.End() - span.SetAttributes(attribute.Int("lim", lim), attribute.Int("shardCount", shardCount)) + span.SetAttributes( + attribute.Int("lim", lim), + attribute.Int("shardCount", shardCount), + attribute.Bool("fast", fast), + ) if shardCount == 0 { shardCount = 50 @@ -217,7 +236,7 @@ func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shar span.SetAttributes(attribute.Int("clampedShardCount", shardCount)) - log := log.With("source", "compactor_enqueue_all_repos") + log := log.With("source", "compactor_enqueue_all_repos", "lim", lim, "shardCount", shardCount, "fast", fast) log.Warn("enqueueing all repos") repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, shardCount) @@ -234,7 +253,7 @@ func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shar span.SetAttributes(attribute.Int("clampedRepos", len(repos))) for _, r := range repos { - c.q.Append(r.Usr) + c.q.Append(r.Usr, fast) } log.Warn("done enqueueing all repos") -- 2.51.2 From 7df3f33f6b9fcc9e94d4969dd22a98173bfa2f95 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 18:35:00 -0700 Subject: [PATCH 150/192] palomar: less verbose query logging --- search/query.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/query.go b/search/query.go index 04f3a23c..37876c57 100644 --- a/search/query.go +++ b/search/query.go @@ -266,7 +266,7 @@ func doSearch(ctx context.Context, escli *es.Client, index string, query interfa if err != nil { return nil, fmt.Errorf("failed to serialize query: %w", err) } - slog.Warn("sending query", "index", index, "query", string(b)) + slog.Info("sending query", "index", index, "query", string(b)) // Perform the search request. res, err := escli.Search( -- 2.51.2 From f7715858344c0ec50e8bce2b5912d0b7f20b66c3 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 18:35:31 -0700 Subject: [PATCH 151/192] palomar: default to at least one replica --- search/post_schema.json | 2 +- search/profile_schema.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/search/post_schema.json b/search/post_schema.json index 46e6f6ec..b764a317 100644 --- a/search/post_schema.json +++ b/search/post_schema.json @@ -2,7 +2,7 @@ "settings": { "index": { "number_of_shards": 6, - "number_of_replicas": 0, + "number_of_replicas": 1, "analysis": { "analyzer": { "default": { diff --git a/search/profile_schema.json b/search/profile_schema.json index c0a6c273..77aa098b 100644 --- a/search/profile_schema.json +++ b/search/profile_schema.json @@ -2,7 +2,7 @@ "settings": { "index": { "number_of_shards": 1, - "number_of_replicas": 0, + "number_of_replicas": 1, "analysis": { "analyzer": { "default": { -- 2.51.2 From 026ed9968fb6de33fb7a895b41ce4536733f8616 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 18:36:24 -0700 Subject: [PATCH 152/192] palomar: profile schema add missing record_cid --- search/profile_schema.json | 1 + 1 file changed, 1 insertion(+) diff --git a/search/profile_schema.json b/search/profile_schema.json index 77aa098b..abd0cc36 100644 --- a/search/profile_schema.json +++ b/search/profile_schema.json @@ -44,6 +44,7 @@ "doc_index_ts": { "type": "date" }, "did": { "type": "keyword", "normalizer": "default", "doc_values": false }, "handle": { "type": "keyword", "normalizer": "default", "copy_to": ["everything", "typeahead"] }, + "record_cid": { "type": "keyword", "normalizer": "default", "doc_values": false }, "display_name": { "type": "text", "analyzer": "textIcu", "search_analyzer": "textIcuSearch", "copy_to": ["everything", "typeahead"] }, "description": { "type": "text", "analyzer": "textIcu", "search_analyzer": "textIcuSearch", "copy_to": "everything" }, -- 2.51.2 From 8e0d26854053f4265a621d7c05438b5a80c7ef3e Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 19:23:27 -0700 Subject: [PATCH 153/192] palomar: bump default refresh interval to 5sec --- search/post_schema.json | 1 + search/profile_schema.json | 1 + 2 files changed, 2 insertions(+) diff --git a/search/post_schema.json b/search/post_schema.json index b764a317..6e03cc39 100644 --- a/search/post_schema.json +++ b/search/post_schema.json @@ -3,6 +3,7 @@ "index": { "number_of_shards": 6, "number_of_replicas": 1, + "refresh_interval": "5s", "analysis": { "analyzer": { "default": { diff --git a/search/profile_schema.json b/search/profile_schema.json index abd0cc36..29c2b665 100644 --- a/search/profile_schema.json +++ b/search/profile_schema.json @@ -3,6 +3,7 @@ "index": { "number_of_shards": 1, "number_of_replicas": 1, + "refresh_interval": "5s", "analysis": { "analyzer": { "default": { -- 2.51.2 From b3509028690f0212085abf85d7f08cf42747584e Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 19:30:55 -0700 Subject: [PATCH 154/192] palomar: keep query fields simple --- search/query.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/search/query.go b/search/query.go index 37876c57..0343ef47 100644 --- a/search/query.go +++ b/search/query.go @@ -97,12 +97,8 @@ func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Cli queryStr, filters := ParseQuery(ctx, dir, q) basic := map[string]interface{}{ "simple_query_string": map[string]interface{}{ - "query": queryStr, - "fields": []string{ - "handle^3", - "display_name^2", - "everything", - }, + "query": queryStr, + "fields": []string{"everything"}, "flags": "AND|NOT|OR|PHRASE|PRECEDENCE|WHITESPACE", "default_operator": "and", "lenient": true, @@ -186,7 +182,6 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s "type": "bool_prefix", "operator": "and", "fields": []string{ - "handle^2", "typeahead", "typeahead._2gram", "typeahead._3gram", -- 2.51.2 From 6b2a28a690d072b9a5ef94ffc670f8baec7e309d Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 19:33:00 -0700 Subject: [PATCH 155/192] palomar: refine rescore boosting --- search/query.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/search/query.go b/search/query.go index 0343ef47..21d47ee8 100644 --- a/search/query.go +++ b/search/query.go @@ -211,6 +211,12 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s "rescore_query": map[string]interface{}{ "boosting": map[string]interface{}{ "positive": map[string]interface{}{ + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q, + }, + }, + // additional boost if it is the full first name "prefix": map[string]interface{}{ "handle": map[string]interface{}{ "value": q + ".", -- 2.51.2 From 4350961a1b187b833dd4ba13a9105d869b6e0f17 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 19:43:09 -0700 Subject: [PATCH 156/192] palomar: tweak rescore boosting for no-handle query --- search/query.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/search/query.go b/search/query.go index 21d47ee8..d7870f80 100644 --- a/search/query.go +++ b/search/query.go @@ -210,16 +210,20 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s "query": map[string]interface{}{ "rescore_query": map[string]interface{}{ "boosting": map[string]interface{}{ - "positive": map[string]interface{}{ - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q, + "positive": []map[string]interface{}{ + { + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q, + }, }, }, // additional boost if it is the full first name - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q + ".", + { + "prefix": map[string]interface{}{ + "handle": map[string]interface{}{ + "value": q + ".", + }, }, }, }, -- 2.51.2 From 783d78029d4b91841bb50fe407621cee494760f0 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 10 Oct 2023 19:43:58 -0700 Subject: [PATCH 157/192] palomar: clarify handle removal with comment --- search/query.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/search/query.go b/search/query.go index d7870f80..91dcbba8 100644 --- a/search/query.go +++ b/search/query.go @@ -182,6 +182,8 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s "type": "bool_prefix", "operator": "and", "fields": []string{ + // adding handle here improves relevency but may be too expensive in prod + //"handle^2", "typeahead", "typeahead._2gram", "typeahead._3gram", -- 2.51.2 From df4d3b01009076ec51ccf3901aa8478fc3f08bbd Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 20:38:59 +0000 Subject: [PATCH 158/192] Progress checkin --- bgs/compact.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bgs/compact.go b/bgs/compact.go index 2feaf336..4d0779ac 100644 --- a/bgs/compact.go +++ b/bgs/compact.go @@ -107,6 +107,8 @@ func NewCompactor() *Compactor { q: &queue{ members: make(map[models.Uid]struct{}), }, + state: &CompactorState{}, + exit: make(chan struct{}), } } -- 2.51.2 From 258a62aea13a9c03ea56ed36ffb62e3586e9dcff Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 20:40:00 +0000 Subject: [PATCH 159/192] Update log levels on things debug-flavored to Debug --- bgs/bgs.go | 12 ++++---- events/schedulers/autoscaling/autoscaling.go | 14 ++++----- indexer/indexer.go | 30 ++++++++++++++------ repomgr/repomgr.go | 2 +- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 30eee5df..37d16e13 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -756,7 +756,7 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event case env.RepoCommit != nil: repoCommitsReceivedCounter.WithLabelValues(host.Host).Add(1) evt := env.RepoCommit - log.Infow("bgs got repo append event", "seq", evt.Seq, "host", host.Host, "repo", evt.Repo) + log.Debugw("bgs got repo append event", "seq", evt.Seq, "host", host.Host, "repo", evt.Repo) u, err := bgs.lookupUserByDid(ctx, evt.Repo) if err != nil { if !errors.Is(err, gorm.ErrRecordNotFound) { @@ -775,7 +775,7 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event } if u.TakenDown { - log.Infow("dropping event from taken down user", "did", evt.Repo, "seq", evt.Seq, "host", host.Host) + log.Debugw("dropping event from taken down user", "did", evt.Repo, "seq", evt.Seq, "host", host.Host) return nil } @@ -784,7 +784,7 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event } if host.ID != u.PDS { - log.Infow("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host) + log.Warnw("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host) subj, err := bgs.createExternalUser(ctx, evt.Repo) if err != nil { return err @@ -974,7 +974,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor externalUserCreationAttempts.Inc() - log.Infof("create external user: %s", did) + log.Debugf("create external user: %s", did) doc, err := s.didr.GetDocument(ctx, did) if err != nil { return nil, fmt.Errorf("could not locate DID document for followed user (%s): %w", did, err) @@ -1054,7 +1054,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor return nil, err } - log.Infow("creating external user", "did", did, "handle", hurl.Host, "pds", peering.ID) + log.Debugw("creating external user", "did", did, "handle", hurl.Host, "pds", peering.ID) handle := hurl.Host @@ -1076,7 +1076,7 @@ func (s *BGS) createExternalUser(ctx context.Context, did string) (*models.Actor exu, err := s.Index.LookupUserByDid(ctx, did) if err == nil { - log.Infow("lost the race to create a new user", "did", did, "handle", handle, "existing_hand", exu.Handle) + log.Debugw("lost the race to create a new user", "did", did, "handle", handle, "existing_hand", exu.Handle) if exu.PDS != peering.ID { // User is now on a different PDS, update if err := s.db.Model(User{}).Where("id = ?", exu.ID).Update("pds", peering.ID).Error; err != nil { diff --git a/events/schedulers/autoscaling/autoscaling.go b/events/schedulers/autoscaling/autoscaling.go index 2376f6a4..04f973bd 100644 --- a/events/schedulers/autoscaling/autoscaling.go +++ b/events/schedulers/autoscaling/autoscaling.go @@ -111,28 +111,28 @@ func NewScheduler(autoscaleSettings AutoscaleSettings, ident string, do func(con } func (p *Scheduler) Shutdown() { - log.Infof("shutting down autoscaling scheduler for %s", p.ident) + log.Debugf("shutting down autoscaling scheduler for %s", p.ident) // stop autoscaling p.autoscalerIn <- struct{}{} close(p.autoscalerIn) <-p.autoscalerOut - log.Info("stopping autoscaling scheduler workers") + log.Debug("stopping autoscaling scheduler workers") // stop workers for i := 0; i < p.concurrency; i++ { p.feeder <- &consumerTask{signal: "stop"} } close(p.feeder) - log.Info("waiting for autoscaling scheduler workers to stop") + log.Debug("waiting for autoscaling scheduler workers to stop") p.workerGroup.Wait() - log.Info("stopping autoscaling scheduler throughput manager") + log.Debug("stopping autoscaling scheduler throughput manager") p.throughputManager.Stop() - log.Info("autoscaling scheduler shutdown complete") + log.Debug("autoscaling scheduler shutdown complete") } // Add autoscaling function @@ -197,7 +197,7 @@ func (p *Scheduler) AddWork(ctx context.Context, repo string, val *events.XRPCSt } func (p *Scheduler) worker() { - log.Infof("starting autoscaling worker for %s", p.ident) + log.Debugf("starting autoscaling worker for %s", p.ident) p.workersActive.Inc() p.workerGroup.Add(1) defer p.workerGroup.Done() @@ -205,7 +205,7 @@ func (p *Scheduler) worker() { for work != nil { // Check if the work item contains a signal to stop the worker. if work.signal == "stop" { - log.Infof("stopping autoscaling worker for %s", p.ident) + log.Debugf("stopping autoscaling worker for %s", p.ident) p.workersActive.Dec() return } diff --git a/indexer/indexer.go b/indexer/indexer.go index f47fc05a..9a89c9cc 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -258,7 +258,7 @@ func (ix *Indexer) crawlRecordReferences(ctx context.Context, op *repomgr.RepoOp case *bsky.FeedLike: if rec.Subject != nil { if err := ix.crawlAtUriRef(ctx, rec.Subject.Uri); err != nil { - log.Infow("failed to crawl vote subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err) + log.Infow("failed to crawl like subject", "cid", op.RecCid, "subjecturi", rec.Subject.Uri, "err", err) } } return nil @@ -324,7 +324,7 @@ func (ix *Indexer) createMissingUserRecord(ctx context.Context, did string) (*mo } func (ix *Indexer) addUserToCrawler(ctx context.Context, ai *models.ActorInfo) error { - log.Infow("Sending user to crawler: ", "did", ai.Did) + log.Debugw("Sending user to crawler: ", "did", ai.Did) if ix.Crawler == nil { return nil } @@ -415,12 +415,18 @@ func (ix *Indexer) fetchRepo(ctx context.Context, c *xrpc.Client, pds *models.PD ctx, span := otel.Tracer("indexer").Start(ctx, "fetchRepo") defer span.End() + span.SetAttributes( + attribute.String("pds", pds.Host), + attribute.String("did", did), + attribute.String("rev", rev), + ) + limiter := ix.GetOrCreateLimiter(pds.ID, pds.CrawlRateLimit) // Wait to prevent DOSing the PDS when connecting to a new stream with lots of active repos limiter.Wait(ctx) - log.Infow("SyncGetRepo", "did", did, "since", rev) + log.Debugw("SyncGetRepo", "did", did, "since", rev) // TODO: max size on these? A malicious PDS could just send us a petabyte sized repo here and kill us repo, err := comatproto.SyncGetRepo(ctx, c, did, rev) if err != nil { @@ -521,7 +527,7 @@ func (ix *Indexer) GetPost(ctx context.Context, uri string) (*models.FeedPost, e } func (ix *Indexer) handleRecordDelete(ctx context.Context, evt *repomgr.RepoEvent, op *repomgr.RepoOp, local bool) error { - log.Infow("record delete event", "collection", op.Collection) + log.Debugw("record delete event", "collection", op.Collection) switch op.Collection { case "app.bsky.feed.post": @@ -611,7 +617,7 @@ func (ix *Indexer) handleRecordDeleteGraphFollow(ctx context.Context, evt *repom } func (ix *Indexer) handleRecordCreate(ctx context.Context, evt *repomgr.RepoEvent, op *repomgr.RepoOp, local bool) ([]uint, error) { - log.Infow("record create event", "collection", op.Collection) + log.Debugw("record create event", "collection", op.Collection) var out []uint switch rec := op.Record.(type) { @@ -652,8 +658,16 @@ func (ix *Indexer) handleRecordCreate(ctx context.Context, evt *repomgr.RepoEven return nil, ix.handleRecordCreateFeedLike(ctx, rec, evt, op) case *bsky.GraphFollow: return out, ix.handleRecordCreateGraphFollow(ctx, rec, evt, op) + case *bsky.GraphBlock: + return out, nil + case *bsky.GraphList: + return out, nil + case *bsky.GraphListitem: + return out, nil + case *bsky.FeedGenerator: + return out, nil case *bsky.ActorProfile: - log.Infof("TODO: got actor profile record creation, need to do something with this") + log.Debugf("TODO: got actor profile record creation, need to do something with this") default: return nil, fmt.Errorf("unrecognized record type: %T", rec) } @@ -727,7 +741,7 @@ func (ix *Indexer) handleRecordCreateGraphFollow(ctx context.Context, rec *bsky. } func (ix *Indexer) handleRecordUpdate(ctx context.Context, evt *repomgr.RepoEvent, op *repomgr.RepoOp, local bool) error { - log.Infow("record update event", "collection", op.Collection) + log.Debugw("record update event", "collection", op.Collection) switch rec := op.Record.(type) { case *bsky.FeedPost: @@ -811,7 +825,7 @@ func (ix *Indexer) handleRecordUpdate(ctx context.Context, evt *repomgr.RepoEven return ix.handleRecordCreateGraphFollow(ctx, rec, evt, op) case *bsky.ActorProfile: - log.Infof("TODO: got actor profile record update, need to do something with this") + log.Debugf("TODO: got actor profile record update, need to do something with this") default: return fmt.Errorf("unrecognized record type: %T", rec) } diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index 1a5d07d5..ba93bc75 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -493,7 +493,7 @@ func (rm *RepoManager) HandleExternalUserEvent(ctx context.Context, pdsid uint, span.SetAttributes(attribute.Int64("uid", int64(uid))) - log.Infow("HandleExternalUserEvent", "pds", pdsid, "uid", uid, "since", since, "nrev", nrev) + log.Debugw("HandleExternalUserEvent", "pds", pdsid, "uid", uid, "since", since, "nrev", nrev) unlock := rm.lockUser(ctx, uid) defer unlock() -- 2.51.2 From f9180356da65ba83d797bdc8481aec93a841fc11 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 11 Oct 2023 13:16:09 -0700 Subject: [PATCH 160/192] palomar: remove query re-scoring --- search/query.go | 99 ------------------------------------------------- 1 file changed, 99 deletions(-) diff --git a/search/query.go b/search/query.go index 91dcbba8..082abb4c 100644 --- a/search/query.go +++ b/search/query.go @@ -7,7 +7,6 @@ import ( "fmt" "io/ioutil" "log/slog" - "strings" "github.com/bluesky-social/indigo/atproto/identity" @@ -123,50 +122,6 @@ func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Cli "from": offset, } - // special-case: exact string match of handle - if strings.HasPrefix(q, "@") && !strings.Contains(q, " ") { - q = q[1:] - query["query"] = map[string]interface{}{ - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q, - }, - }, - } - } - - // boost exact handle prefix match, if q is single simple term - if len(q) >= 3 && !strings.ContainsAny(q, " .") { - query["rescore"] = map[string]interface{}{ - "window_size": 100, - "query": map[string]interface{}{ - "rescore_query": map[string]interface{}{ - "boosting": map[string]interface{}{ - "positive": map[string]interface{}{ - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q + ".", - }, - }, - }, - // downrank *.bsky.social (vs custom domain) - // wildcard is expensive, so only in rescore - "negative": map[string]interface{}{ - "wildcard": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": "*.bsky.social", - }, - }, - }, - "negative_boost": 0.5, - }, - }, - "query_weight": 0.5, - "rescore_query_weight": 2.0, - }, - } - } - return doSearch(ctx, escli, index, query) } @@ -193,60 +148,6 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s "size": size, } - // special-case: exact string match of handle - if strings.HasPrefix(q, "@") && !strings.Contains(q, " ") { - q = q[1:] - query["query"] = map[string]interface{}{ - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q, - }, - }, - } - } - - // boost exact handle prefix match, if q is single simple term - if len(q) >= 3 && !strings.ContainsAny(q, " .") { - query["rescore"] = map[string]interface{}{ - "window_size": 100, - "query": map[string]interface{}{ - "rescore_query": map[string]interface{}{ - "boosting": map[string]interface{}{ - "positive": []map[string]interface{}{ - { - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q, - }, - }, - }, - // additional boost if it is the full first name - { - "prefix": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": q + ".", - }, - }, - }, - }, - // downrank *.bsky.social (vs custom domain) - // wildcard is expensive, so only in rescore - "negative": map[string]interface{}{ - "wildcard": map[string]interface{}{ - "handle": map[string]interface{}{ - "value": "*.bsky.social", - }, - }, - }, - "negative_boost": 0.5, - }, - }, - "query_weight": 1.0, - "rescore_query_weight": 1.0, - }, - } - } - return doSearch(ctx, escli, index, query) } -- 2.51.2 From ae75b62db599ef07cfb630730f6befa8c7b52568 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 21:06:34 +0000 Subject: [PATCH 161/192] Add tracing to Palomar queries --- cmd/palomar/main.go | 42 ++++++++++++++++++ go.mod | 37 ++++++++-------- go.sum | 105 +++++++++++++++++--------------------------- search/handlers.go | 40 +++++++++++++++-- search/query.go | 18 ++++++++ search/server.go | 3 +- 6 files changed, 157 insertions(+), 88 deletions(-) diff --git a/cmd/palomar/main.go b/cmd/palomar/main.go index 2743afee..3d88edc0 100644 --- a/cmd/palomar/main.go +++ b/cmd/palomar/main.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "encoding/json" "fmt" + "log" "log/slog" "net/http" "os" @@ -12,6 +13,12 @@ import ( "time" _ "github.com/joho/godotenv/autoload" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.4.0" "golang.org/x/time/rate" "github.com/bluesky-social/indigo/atproto/identity" @@ -158,6 +165,41 @@ var runCmd = &cli.Command{ })) slog.SetDefault(logger) + // Enable OTLP HTTP exporter + // For relevant environment variables: + // https://pkg.go.dev/go.opentelemetry.io/otel/exporters/otlp/otlptrace#readme-environment-variables + // At a minimum, you need to set + // OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 + if ep := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT"); ep != "" { + slog.Info("setting up trace exporter", "endpoint", ep) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + exp, err := otlptracehttp.New(ctx) + if err != nil { + log.Fatal("failed to create trace exporter", "error", err) + } + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := exp.Shutdown(ctx); err != nil { + slog.Error("failed to shutdown trace exporter", "error", err) + } + }() + + tp := tracesdk.NewTracerProvider( + tracesdk.WithBatcher(exp), + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNameKey.String("palomar"), + attribute.String("env", os.Getenv("ENVIRONMENT")), // DataDog + attribute.String("environment", os.Getenv("ENVIRONMENT")), // Others + attribute.Int64("ID", 1), + )), + ) + otel.SetTracerProvider(tp) + } + db, err := cliutil.SetupDatabase(cctx.String("database-url"), cctx.Int("max-metadb-connections")) if err != nil { return err diff --git a/go.mod b/go.mod index 0223f3c8..9ecb7ff9 100644 --- a/go.mod +++ b/go.mod @@ -55,18 +55,18 @@ require ( github.com/whyrusleeping/go-did v0.0.0-20230824162731-404d1707d5d6 gitlab.com/yawning/secp256k1-voi v0.0.0-20230815035612-a7264edccf80 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 - go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/exporters/jaeger v1.14.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 - go.opentelemetry.io/otel/sdk v1.16.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 + go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/trace v1.19.0 go.uber.org/automaxprocs v1.5.3 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.13.0 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 - golang.org/x/sync v0.1.0 - golang.org/x/text v0.12.0 + golang.org/x/sync v0.3.0 + golang.org/x/text v0.13.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.8.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 @@ -95,7 +95,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/ipfs/bbloom v0.0.4 // indirect @@ -147,19 +147,18 @@ require ( github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect - go.opentelemetry.io/otel/metric v1.16.0 // indirect - go.opentelemetry.io/proto/otlp v0.19.0 // indirect + go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.45.0 + go.opentelemetry.io/otel/metric v1.19.0 // indirect + go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/sys v0.11.0 // indirect - google.golang.org/genproto v0.0.0-20230526015343-6ee61e4f9d5f // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230526161137-0005af68ea54 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230526161137-0005af68ea54 // indirect - google.golang.org/grpc v1.55.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/net v0.15.0 // indirect + golang.org/x/sys v0.12.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect + google.golang.org/grpc v1.58.2 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index cbc69fec..b2aaa467 100644 --- a/go.sum +++ b/go.sum @@ -35,7 +35,6 @@ contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -44,7 +43,6 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 h1:iW0a5ljuFxkLGPNem5Ui+KBjFJzKg4Fv2fnxe4dvzpM= github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5/go.mod h1:Y2QMoi1vgtOIfc+6DhrMOGkLoGzqSV2rKp4Sm+opsyA= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de h1:FxWPpzIjnTlhPwqqXc4/vE0f7GvRjuAsbW+HOIe8KnA= github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= github.com/aws/aws-sdk-go v1.44.180/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= @@ -78,7 +76,6 @@ github.com/carlmjohnson/versioninfo v0.22.5/go.mod h1:QT9mph3wcVfISUKd0i9sZfVrPv github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -88,12 +85,6 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/corpix/uarand v0.0.0-20170723150923-031be390f409 h1:9A+mfQmwzZ6KwUXPc8nHxFtKgn9VIvO3gXAOspIcE3s= github.com/corpix/uarand v0.0.0-20170723150923-031be390f409/go.mod h1:JSm890tOkDN+M1jqN8pUGDKnzJrsVbJwSMHBY4zwz7M= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -113,9 +104,6 @@ github.com/dustinkirkland/golang-petname v0.0.0-20230626224747-e794b9370d49/go.m github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= -github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -124,7 +112,6 @@ github.com/flosch/pongo2/v6 v6.0.0/go.mod h1:CuDpFm47R0uGGE7z13/tTlt1Y6zdxvr2RLT github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -157,7 +144,6 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69 github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -205,7 +191,6 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -234,10 +219,8 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGa github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 h1:gDLXvp5S9izjldquuoAhDzccbskOL6tDC5jMSyx3zxE= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2/go.mod h1:7pdNwVWBBHGiCxa9lAszqCJMbfTISJ7oMftp8+UGV08= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= @@ -561,7 +544,6 @@ github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -590,7 +572,6 @@ github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYl github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -601,7 +582,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -651,27 +631,28 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.45.0 h1:JJCIHAxGCB5HM3NxeIwFjHc087Xwk96TG9kaZU6TAec= +go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho v0.45.0/go.mod h1:Px9kH7SJ+NhsgWRtD/eMcs15Tyt4uL3rM7X54qv6pfA= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0 h1:pginetY7+onl4qN1vl0xW/V/v6OBZ0vVdH+esuJgvmM= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.42.0/go.mod h1:XiYsayHc36K3EByOO6nbAXnAWbrUxdjUROCEeeROOH8= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/contrib/propagators/b3 v1.20.0 h1:Yty9Vs4F3D6/liF1o6FNt0PvN85h/BJJ6DQKJ3nrcM0= +go.opentelemetry.io/contrib/propagators/b3 v1.20.0/go.mod h1:On4VgbkqYL18kbJlWsa18+cMNe6rYpBnPi1ARI/BrsU= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/exporters/jaeger v1.14.0 h1:CjbUNd4iN2hHmWekmOqZ+zSCU+dzZppG8XsV+A3oc8Q= go.opentelemetry.io/otel/exporters/jaeger v1.14.0/go.mod h1:4Ay9kk5vELRrbg5z4cpP9EtmQRFap2Wb0woPG4lujZA= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -704,8 +685,9 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -782,15 +764,14 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -805,8 +786,9 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -864,8 +846,9 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -878,14 +861,14 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -993,20 +976,18 @@ google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20230526015343-6ee61e4f9d5f h1:DwRdHa3+SynqBR2tx3LVtzJrGooL9hg1OCAfBdQAk1A= -google.golang.org/genproto v0.0.0-20230526015343-6ee61e4f9d5f/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= -google.golang.org/genproto/googleapis/api v0.0.0-20230526161137-0005af68ea54 h1:VW/GdnI343CnUKGap8QyxQ204yai8uUsGzGmyOFfwH8= -google.golang.org/genproto/googleapis/api v0.0.0-20230526161137-0005af68ea54/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230526161137-0005af68ea54 h1:wQvmPUaH4JVFCzNAL9ShNjezVoq3OhlinNMLYSAN9Vg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230526161137-0005af68ea54/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= +google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= +google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -1019,13 +1000,9 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1038,11 +1015,10 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1054,7 +1030,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/search/handlers.go b/search/handlers.go index b935e0ee..a1afd5d4 100644 --- a/search/handlers.go +++ b/search/handlers.go @@ -12,8 +12,12 @@ import ( "github.com/labstack/echo/v4" otel "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" ) +var tracer = otel.Tracer("search") + func parseCursorLimit(e echo.Context) (int, int, error) { offset := 0 if c := strings.TrimSpace(e.QueryParam("cursor")); c != "" { @@ -60,9 +64,11 @@ func parseCursorLimit(e echo.Context) (int, int, error) { } func (s *Server) handleSearchPostsSkeleton(e echo.Context) error { - ctx, span := otel.Tracer("search").Start(e.Request().Context(), "handleSearchPostsSkeleton") + ctx, span := tracer.Start(e.Request().Context(), "handleSearchPostsSkeleton") defer span.End() + span.SetAttributes(attribute.String("query", e.QueryParam("q"))) + q := strings.TrimSpace(e.QueryParam("q")) if q == "" { return e.JSON(400, map[string]any{ @@ -72,21 +78,31 @@ func (s *Server) handleSearchPostsSkeleton(e echo.Context) error { offset, limit, err := parseCursorLimit(e) if err != nil { + span.SetAttributes(attribute.String("error", fmt.Sprintf("invalid cursor/limit: %s", err))) + span.SetStatus(codes.Error, err.Error()) return err } + span.SetAttributes(attribute.Int("offset", offset), attribute.Int("limit", limit)) + out, err := s.SearchPosts(ctx, q, offset, limit) if err != nil { + span.SetAttributes(attribute.String("error", fmt.Sprintf("failed to SearchPosts: %s", err))) + span.SetStatus(codes.Error, err.Error()) return err } + span.SetAttributes(attribute.Int("posts.length", len(out.Posts))) + return e.JSON(200, out) } func (s *Server) handleSearchActorsSkeleton(e echo.Context) error { - ctx, span := otel.Tracer("search").Start(e.Request().Context(), "handleSearchActorsSkeleton") + ctx, span := tracer.Start(e.Request().Context(), "handleSearchActorsSkeleton") defer span.End() + span.SetAttributes(attribute.String("query", e.QueryParam("q"))) + q := strings.TrimSpace(e.QueryParam("q")) if q == "" { return e.JSON(400, map[string]any{ @@ -96,6 +112,8 @@ func (s *Server) handleSearchActorsSkeleton(e echo.Context) error { offset, limit, err := parseCursorLimit(e) if err != nil { + span.SetAttributes(attribute.String("error", fmt.Sprintf("invalid cursor/limit: %s", err))) + span.SetStatus(codes.Error, err.Error()) return err } @@ -104,11 +122,21 @@ func (s *Server) handleSearchActorsSkeleton(e echo.Context) error { typeahead = true } + span.SetAttributes( + attribute.Int("offset", offset), + attribute.Int("limit", limit), + attribute.Bool("typeahead", typeahead), + ) + out, err := s.SearchProfiles(ctx, q, typeahead, offset, limit) if err != nil { + span.SetAttributes(attribute.String("error", fmt.Sprintf("failed to SearchProfiles: %s", err))) + span.SetStatus(codes.Error, err.Error()) return err } + span.SetAttributes(attribute.Int("actors.length", len(out.Actors))) + return e.JSON(200, out) } @@ -118,7 +146,7 @@ type IndexError struct { } func (s *Server) handleIndexRepos(e echo.Context) error { - ctx, span := otel.Tracer("search").Start(e.Request().Context(), "handleIndexRepos") + ctx, span := tracer.Start(e.Request().Context(), "handleIndexRepos") defer span.End() dids, ok := e.QueryParams()["did"] @@ -166,6 +194,9 @@ func (s *Server) handleIndexRepos(e echo.Context) error { } func (s *Server) SearchPosts(ctx context.Context, q string, offset, size int) (*appbsky.UnspeccedSearchPostsSkeleton_Output, error) { + ctx, span := tracer.Start(ctx, "SearchPosts") + defer span.End() + resp, err := DoSearchPosts(ctx, s.dir, s.escli, s.postIndex, q, offset, size) if err != nil { return nil, err @@ -201,6 +232,9 @@ func (s *Server) SearchPosts(ctx context.Context, q string, offset, size int) (* } func (s *Server) SearchProfiles(ctx context.Context, q string, typeahead bool, offset, size int) (*appbsky.UnspeccedSearchActorsSkeleton_Output, error) { + ctx, span := tracer.Start(ctx, "SearchProfiles") + defer span.End() + var resp *EsSearchResponse var err error if typeahead { diff --git a/search/query.go b/search/query.go index 082abb4c..9bbb384b 100644 --- a/search/query.go +++ b/search/query.go @@ -9,6 +9,7 @@ import ( "log/slog" "github.com/bluesky-social/indigo/atproto/identity" + "go.opentelemetry.io/otel/attribute" es "github.com/opensearch-project/opensearch-go/v2" ) @@ -55,6 +56,9 @@ func checkParams(offset, size int) error { } func DoSearchPosts(ctx context.Context, dir identity.Directory, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) { + ctx, span := tracer.Start(ctx, "DoSearchPosts") + defer span.End() + if err := checkParams(offset, size); err != nil { return nil, err } @@ -89,6 +93,9 @@ func DoSearchPosts(ctx context.Context, dir identity.Directory, escli *es.Client } func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Client, index, q string, offset, size int) (*EsSearchResponse, error) { + ctx, span := tracer.Start(ctx, "DoSearchProfiles") + defer span.End() + if err := checkParams(offset, size); err != nil { return nil, err } @@ -126,6 +133,9 @@ func DoSearchProfiles(ctx context.Context, dir identity.Directory, escli *es.Cli } func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q string, size int) (*EsSearchResponse, error) { + ctx, span := tracer.Start(ctx, "DoSearchProfilesTypeahead") + defer span.End() + if err := checkParams(0, size); err != nil { return nil, err } @@ -153,6 +163,9 @@ func DoSearchProfilesTypeahead(ctx context.Context, escli *es.Client, index, q s // helper to do a full-featured Lucene query parser (query_string) search, with all possible facets. Not safe to expose publicly. func DoSearchGeneric(ctx context.Context, escli *es.Client, index, q string) (*EsSearchResponse, error) { + ctx, span := tracer.Start(ctx, "DoSearchGeneric") + defer span.End() + query := map[string]interface{}{ "query": map[string]interface{}{ "query_string": map[string]interface{}{ @@ -170,6 +183,11 @@ func DoSearchGeneric(ctx context.Context, escli *es.Client, index, q string) (*E } func doSearch(ctx context.Context, escli *es.Client, index string, query interface{}) (*EsSearchResponse, error) { + ctx, span := tracer.Start(ctx, "doSearch") + defer span.End() + + span.SetAttributes(attribute.String("index", index), attribute.String("query", fmt.Sprintf("%+v", query))) + b, err := json.Marshal(query) if err != nil { return nil, fmt.Errorf("failed to serialize query: %w", err) diff --git a/search/server.go b/search/server.go index 2106104f..4dba01d8 100644 --- a/search/server.go +++ b/search/server.go @@ -15,6 +15,7 @@ import ( "github.com/bluesky-social/indigo/util/version" "github.com/bluesky-social/indigo/xrpc" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/contrib/instrumentation/github.com/labstack/echo/otelecho" "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" @@ -53,7 +54,6 @@ type Config struct { } func NewServer(db *gorm.DB, escli *es.Client, dir identity.Directory, config Config) (*Server, error) { - logger := config.Logger if logger == nil { logger = slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ @@ -186,6 +186,7 @@ func (s *Server) RunAPI(listen string) error { e.Use(middleware.Recover()) e.Use(MetricsMiddleware) e.Use(middleware.BodyLimit("64M")) + e.Use(otelecho.Middleware("palomar")) e.HTTPErrorHandler = func(err error, ctx echo.Context) { code := 500 -- 2.51.2 From 374dbef47b284bec9ea01a1f2e5d447cd4ac0eca Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 21:10:38 +0000 Subject: [PATCH 162/192] Add some traces to indexing too --- search/indexing.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/search/indexing.go b/search/indexing.go index f4851cd6..922a037b 100644 --- a/search/indexing.go +++ b/search/indexing.go @@ -14,11 +14,16 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/util" "github.com/ipfs/go-cid" + "go.opentelemetry.io/otel/attribute" esapi "github.com/opensearch-project/opensearch-go/v2/opensearchapi" ) func (s *Server) deletePost(ctx context.Context, ident *identity.Identity, rkey string) error { + ctx, span := tracer.Start(ctx, "deletePost") + defer span.End() + span.SetAttributes(attribute.String("repo", ident.DID.String()), attribute.String("rkey", rkey)) + log := s.logger.With("repo", ident.DID, "rkey", rkey, "op", "deletePost") log.Info("deleting post from index") docID := fmt.Sprintf("%s_%s", ident.DID.String(), rkey) @@ -45,6 +50,10 @@ func (s *Server) deletePost(ctx context.Context, ident *identity.Identity, rkey } func (s *Server) indexPost(ctx context.Context, ident *identity.Identity, rec *appbsky.FeedPost, path string, rcid cid.Cid) error { + ctx, span := tracer.Start(ctx, "indexPost") + defer span.End() + span.SetAttributes(attribute.String("repo", ident.DID.String()), attribute.String("path", path)) + log := s.logger.With("repo", ident.DID, "path", path, "op", "indexPost") parts := strings.SplitN(path, "/", 3) // TODO: replace with an atproto/syntax package type for TID @@ -95,6 +104,10 @@ func (s *Server) indexPost(ctx context.Context, ident *identity.Identity, rec *a } func (s *Server) indexProfile(ctx context.Context, ident *identity.Identity, rec *appbsky.ActorProfile, path string, rcid cid.Cid) error { + ctx, span := tracer.Start(ctx, "indexProfile") + defer span.End() + span.SetAttributes(attribute.String("repo", ident.DID.String()), attribute.String("path", path)) + log := s.logger.With("repo", ident.DID, "path", path, "op", "indexProfile") parts := strings.SplitN(path, "/", 3) if len(parts) != 2 || parts[1] != "self" { @@ -134,6 +147,10 @@ func (s *Server) indexProfile(ctx context.Context, ident *identity.Identity, rec } func (s *Server) updateUserHandle(ctx context.Context, did syntax.DID, handle string) error { + ctx, span := tracer.Start(ctx, "updateUserHandle") + defer span.End() + span.SetAttributes(attribute.String("repo", did.String()), attribute.String("event.handle", handle)) + log := s.logger.With("repo", did.String(), "op", "updateUserHandle", "handle_from_event", handle) err := s.dir.Purge(ctx, did.AtIdentifier()) @@ -154,6 +171,7 @@ func (s *Server) updateUserHandle(ctx context.Context, did syntax.DID, handle st } log.Info("updating user handle", "handle_from_dir", ident.Handle) + span.SetAttributes(attribute.String("dir.handle", ident.Handle.String())) b, err := json.Marshal(map[string]any{ "script": map[string]any{ -- 2.51.2 From a9d33cf44259711b48033ff0bf7508fcee4864de Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 21:22:53 +0000 Subject: [PATCH 163/192] Create top-level contexts in event handlers for indexer --- search/firehose.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/search/firehose.go b/search/firehose.go index 5b71a5ba..913fef17 100644 --- a/search/firehose.go +++ b/search/firehose.go @@ -70,6 +70,10 @@ func (s *Server) RunIndexer(ctx context.Context) error { rsc := &events.RepoStreamCallbacks{ RepoCommit: func(evt *comatproto.SyncSubscribeRepos_Commit) error { + ctx := context.Background() + ctx, span := tracer.Start(ctx, "RepoCommit") + defer span.End() + defer func() { if evt.Seq%50 == 0 { if err := s.updateLastCursor(evt.Seq); err != nil { @@ -153,6 +157,10 @@ func (s *Server) RunIndexer(ctx context.Context) error { }, RepoHandle: func(evt *comatproto.SyncSubscribeRepos_Handle) error { + ctx := context.Background() + ctx, span := tracer.Start(ctx, "RepoHandle") + defer span.End() + did, err := syntax.ParseDID(evt.Did) if err != nil { s.logger.Error("bad DID in RepoHandle event", "did", evt.Did, "handle", evt.Handle, "seq", evt.Seq, "err", err) -- 2.51.2 From 779e954e895f50de2446643386a4447872e6a2cf Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 11 Oct 2023 21:35:51 +0000 Subject: [PATCH 164/192] More otel stuff --- search/firehose.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/search/firehose.go b/search/firehose.go index 913fef17..63eb7bee 100644 --- a/search/firehose.go +++ b/search/firehose.go @@ -18,6 +18,7 @@ import ( lexutil "github.com/bluesky-social/indigo/lex/util" "github.com/bluesky-social/indigo/repo" "github.com/bluesky-social/indigo/repomgr" + "github.com/bluesky-social/indigo/util/version" "github.com/gorilla/websocket" "github.com/ipfs/go-cid" @@ -63,7 +64,9 @@ func (s *Server) RunIndexer(ctx context.Context) error { if cur != 0 { u.RawQuery = fmt.Sprintf("cursor=%d", cur) } - con, _, err := d.Dial(u.String(), http.Header{}) + con, _, err := d.Dial(u.String(), http.Header{ + "User-Agent": []string{fmt.Sprintf("palomar/%s", version.Version)}, + }) if err != nil { return fmt.Errorf("events dial failed: %w", err) } -- 2.51.2 From d0872581144ca6143fd93677675b07a264f97914 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 1 Oct 2023 15:56:10 -0700 Subject: [PATCH 165/192] syntax: basic CID (string only) syntax --- atproto/syntax/cid.go | 48 ++++++++++++++++++ atproto/syntax/cid_test.go | 50 +++++++++++++++++++ .../syntax/testdata/cid_syntax_invalid.txt | 16 ++++++ atproto/syntax/testdata/cid_syntax_valid.txt | 14 ++++++ 4 files changed, 128 insertions(+) create mode 100644 atproto/syntax/cid.go create mode 100644 atproto/syntax/cid_test.go create mode 100644 atproto/syntax/testdata/cid_syntax_invalid.txt create mode 100644 atproto/syntax/testdata/cid_syntax_valid.txt diff --git a/atproto/syntax/cid.go b/atproto/syntax/cid.go new file mode 100644 index 00000000..c9aa0de7 --- /dev/null +++ b/atproto/syntax/cid.go @@ -0,0 +1,48 @@ +package syntax + +import ( + "fmt" + "regexp" + "strings" +) + +// Represents a CIDv1 in string format, as would pass Lexicon syntax validation. +// +// You usually want to use the github.com/ipfs/go-cid package and type when working with CIDs ("Links") in atproto. This specific type (syntax.CID) is an informal/incomplete helper specifically for doing fast string verification or pass-through without parsing, re-serialization, or normalization. +// +// Always use [ParseCID] instead of wrapping strings directly, especially when working with network input. +type CID string + +func ParseCID(raw string) (CID, error) { + if len(raw) > 256 { + return "", fmt.Errorf("CID is too long (256 chars max)") + } + if len(raw) < 8 { + return "", fmt.Errorf("CID is too short (8 chars min)") + } + var cidRegex = regexp.MustCompile(`^[a-zA-Z0-9+=]{8,256}$`) + if !cidRegex.MatchString(raw) { + return "", fmt.Errorf("CID syntax didn't validate via regex") + } + if strings.HasPrefix(raw, "Qmb") { + return "", fmt.Errorf("CIDv0 not allowed in this version of atproto") + } + return CID(raw), nil +} + +func (c CID) String() string { + return string(c) +} + +func (c CID) MarshalText() ([]byte, error) { + return []byte(c.String()), nil +} + +func (c *CID) UnmarshalText(text []byte) error { + cid, err := ParseCID(string(text)) + if err != nil { + return err + } + *c = cid + return nil +} diff --git a/atproto/syntax/cid_test.go b/atproto/syntax/cid_test.go new file mode 100644 index 00000000..7da07c6c --- /dev/null +++ b/atproto/syntax/cid_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropCIDsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/cid_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseCID(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropCIDsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/cid_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseCID(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} diff --git a/atproto/syntax/testdata/cid_syntax_invalid.txt b/atproto/syntax/testdata/cid_syntax_invalid.txt new file mode 100644 index 00000000..5bd1d007 --- /dev/null +++ b/atproto/syntax/testdata/cid_syntax_invalid.txt @@ -0,0 +1,16 @@ +example.com +https://example.com +cid:bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi +. +12345 + +# whitespace + bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi +bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi +bafybe igdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi + +# old CIDv0 not supported +QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR + +# https://github.com/ipfs-shipyard/is-ipfs/blob/master/test/test-cid.spec.ts +noop diff --git a/atproto/syntax/testdata/cid_syntax_valid.txt b/atproto/syntax/testdata/cid_syntax_valid.txt new file mode 100644 index 00000000..9dd3dfed --- /dev/null +++ b/atproto/syntax/testdata/cid_syntax_valid.txt @@ -0,0 +1,14 @@ + +# examples from https://docs.ipfs.tech/concepts/content-addressing +bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi + +# https://github.com/ipfs-shipyard/is-ipfs/blob/master/test/test-cid.spec.ts +zdj7WWeQ43G6JJvLWQWZpyHuAMq6uYWRjkBXFad11vE2LHhQ7 +bafybeie5gq4jxvzmsym6hjlwxej4rwdoxt7wadqvmmwbqi7r27fclha2va + +# more contrived examples +mBcDxtdWx0aWhhc2g+ +z7x3CtScH765HvShXT +zdj7WhuEjrB52m1BisYCtmjH1hSKa7yZ3jEZ9JcXaFRD51wVz +7134036155352661643226414134664076 +f017012202c5f688262e0ece8569aa6f94d60aad55ca8d9d83734e4a7430d0cff6588ec2b -- 2.51.2 From 0bff501b795bc56c4f042725d073494893593332 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 1 Oct 2023 16:03:08 -0700 Subject: [PATCH 166/192] syntax: partial TID type (validation-only) --- .../syntax/testdata/tid_syntax_invalid.txt | 15 ++++++ atproto/syntax/testdata/tid_syntax_valid.txt | 6 +++ atproto/syntax/tid.go | 43 ++++++++++++++++ atproto/syntax/tid_test.go | 50 +++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 atproto/syntax/testdata/tid_syntax_invalid.txt create mode 100644 atproto/syntax/testdata/tid_syntax_valid.txt create mode 100644 atproto/syntax/tid.go create mode 100644 atproto/syntax/tid_test.go diff --git a/atproto/syntax/testdata/tid_syntax_invalid.txt b/atproto/syntax/testdata/tid_syntax_invalid.txt new file mode 100644 index 00000000..eca90b2d --- /dev/null +++ b/atproto/syntax/testdata/tid_syntax_invalid.txt @@ -0,0 +1,15 @@ + +# not base32 +3jzfcijpj2z21 +0000000000000 + +# too long/short +3jzfcijpj2z2aa +3jzfcijpj2z2 + +# old dashes syntax not actually supported (TTTT-TTT-TTTT-CC) +3jzf-cij-pj2z-2a + +# high bit can't be high +zzzzzzzzzzzzz +kjzfcijpj2z2a diff --git a/atproto/syntax/testdata/tid_syntax_valid.txt b/atproto/syntax/testdata/tid_syntax_valid.txt new file mode 100644 index 00000000..b161a3fe --- /dev/null +++ b/atproto/syntax/testdata/tid_syntax_valid.txt @@ -0,0 +1,6 @@ +# 13 digits +# 234567abcdefghijklmnopqrstuvwxyz + +3jzfcijpj2z2a +7777777777777 +3zzzzzzzzzzzz diff --git a/atproto/syntax/tid.go b/atproto/syntax/tid.go new file mode 100644 index 00000000..47f3ebda --- /dev/null +++ b/atproto/syntax/tid.go @@ -0,0 +1,43 @@ +package syntax + +import ( + "fmt" + "regexp" +) + +// Represents a TID in string format, as would pass Lexicon syntax validation. +// +// Always use [ParseTID] instead of wrapping strings directly, especially when working with network input. +// +// Syntax specification: https://atproto.com/specs/record-key +type TID string + +func ParseTID(raw string) (TID, error) { + if len(raw) != 13 { + return "", fmt.Errorf("TID is wrong length (expected 13 chars)") + } + var tidRegex = regexp.MustCompile(`^[234567abcdefghij][234567abcdefghijklmnopqrstuvwxyz]{12}$`) + if !tidRegex.MatchString(raw) { + return "", fmt.Errorf("TID syntax didn't validate via regex") + } + return TID(raw), nil +} + +// TODO: additional helpers: to timestamp, from timestamp, from integer, etc + +func (t TID) String() string { + return string(t) +} + +func (t TID) MarshalText() ([]byte, error) { + return []byte(t.String()), nil +} + +func (t *TID) UnmarshalText(text []byte) error { + tid, err := ParseTID(string(text)) + if err != nil { + return err + } + *t = tid + return nil +} diff --git a/atproto/syntax/tid_test.go b/atproto/syntax/tid_test.go new file mode 100644 index 00000000..9d2c883a --- /dev/null +++ b/atproto/syntax/tid_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropTIDsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/tid_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseTID(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropTIDsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/tid_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseTID(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} -- 2.51.2 From 008311fe4ad4178286421f643c77fa507d495ffe Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 1 Oct 2023 16:36:51 -0700 Subject: [PATCH 167/192] syntax: basic datetime parsing --- atproto/syntax/datetime.go | 43 +++++++++++ atproto/syntax/datetime_test.go | 50 +++++++++++++ .../testdata/datetime_syntax_invalid.txt | 72 +++++++++++++++++++ .../syntax/testdata/datetime_syntax_valid.txt | 20 ++++++ 4 files changed, 185 insertions(+) create mode 100644 atproto/syntax/datetime.go create mode 100644 atproto/syntax/datetime_test.go create mode 100644 atproto/syntax/testdata/datetime_syntax_invalid.txt create mode 100644 atproto/syntax/testdata/datetime_syntax_valid.txt diff --git a/atproto/syntax/datetime.go b/atproto/syntax/datetime.go new file mode 100644 index 00000000..28279cb1 --- /dev/null +++ b/atproto/syntax/datetime.go @@ -0,0 +1,43 @@ +package syntax + +import ( + "fmt" + "regexp" + "strings" +) + +// Represents the a Datetime in string format, as would pass Lexicon syntax validation: the intersection of RFC-3339 and ISO-8601 syntax. +// +// Always use [ParseDatetime] instead of wrapping strings directly, especially when working with network input. +type Datetime string + +func ParseDatetime(raw string) (Datetime, error) { + if len(raw) > 64 { + return "", fmt.Errorf("Datetime too long (max 64 chars)") + } + var datetimeRegex = regexp.MustCompile(`^[0-9]{4}-[01][0-9]-[0-3][0-9]T[0-2][0-9]:[0-6][0-9]:[0-6][0-9](.[0-9]{1,20})?(Z|([+-][0-2][0-9]:[0-5][0-9]))$`) + if !datetimeRegex.MatchString(raw) { + return "", fmt.Errorf("Datetime syntax didn't validate via regex") + } + if strings.HasSuffix(raw, "-00:00") { + return "", fmt.Errorf("Datetime can't use '-00:00' for UTC timezone, must use '+00:00', per ISO-8601") + } + return Datetime(raw), nil +} + +func (d Datetime) String() string { + return string(d) +} + +func (d Datetime) MarshalText() ([]byte, error) { + return []byte(d.String()), nil +} + +func (d *Datetime) UnmarshalText(text []byte) error { + datetime, err := ParseDatetime(string(text)) + if err != nil { + return err + } + *d = datetime + return nil +} diff --git a/atproto/syntax/datetime_test.go b/atproto/syntax/datetime_test.go new file mode 100644 index 00000000..0e93cb17 --- /dev/null +++ b/atproto/syntax/datetime_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropDatetimesValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/datetime_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDatetime(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropDatetimesInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/datetime_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDatetime(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} diff --git a/atproto/syntax/testdata/datetime_syntax_invalid.txt b/atproto/syntax/testdata/datetime_syntax_invalid.txt new file mode 100644 index 00000000..c4a0e764 --- /dev/null +++ b/atproto/syntax/testdata/datetime_syntax_invalid.txt @@ -0,0 +1,72 @@ + +# subtle changes to: 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123z +01985-04-12T23:20:50.123Z +985-04-12T23:20:50.123Z +1985-04-12T23:20:50.Z +1985-04-32T23;20:50.123Z +1985-04-32T23;20:50.123Z + +# en-dash and em-dash +1985—04-32T23;20:50.123Z +1985–04-32T23;20:50.123Z + +# whitespace + 1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.123Z +1985-04-12T 23:20:50.123Z + +# TODO: full parse to validate? +#1985-00-12T23:20:50.123Z +#1985-04-00T23:20:50.123Z +#1985-13-12T23:20:50.123Z +#1985-04-12T25:20:50.123Z +#1985-04-12T23:61:50.123Z +#1985-04-12T23:20:61.123Z + +# not enough zero padding +1985-4-12T23:20:50.123Z +1985-04-2T23:20:50.123Z +1985-04-12T3:20:50.123Z +1985-04-12T23:0:50.123Z +1985-04-12T23:20:5.123Z + +# too much zero padding +01985-04-12T23:20:50.123Z +1985-004-12T23:20:50.123Z +1985-04-012T23:20:50.123Z +1985-04-12T023:20:50.123Z +1985-04-12T23:020:50.123Z +1985-04-12T23:20:050.123Z + +# strict capitalization (ISO-8601) +1985-04-12t23:20:50.123Z +1985-04-12T23:20:50.123z + +# RFC-3339, but not ISO-8601 +1985-04-12T23:20:50.123-00:00 +1985-04-12_23:20:50.123Z +1985-04-12 23:20:50.123Z + +# ISO-8601, but weird +1985-04-274T23:20:50.123Z + +# timezone is required +1985-04-12T23:20:50.123 +1985-04-12T23:20:50 + +1985-04-12 +1985-04-12T23:20Z +1985-04-12T23:20:5Z +1985-04-12T23:20:50.123 ++001985-04-12T23:20:50.123Z +23:20:50.123Z + +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+00:0 +1985-04-12T23:20:50.123+0:00 +1985-04-12T23:20:50.123 +1985-04-12T23:20:50.123+0000 +1985-04-12T23:20:50.123+00 +1985-04-12T23:20:50.123+ +1985-04-12T23:20:50.123- diff --git a/atproto/syntax/testdata/datetime_syntax_valid.txt b/atproto/syntax/testdata/datetime_syntax_valid.txt new file mode 100644 index 00000000..798c7485 --- /dev/null +++ b/atproto/syntax/testdata/datetime_syntax_valid.txt @@ -0,0 +1,20 @@ +# "preferred" +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.000Z +2000-01-01T00:00:00.000Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.120Z +1985-04-12T23:20:50.120000Z + +# "supported" +1985-04-12T23:20:50.1235678912345Z +1985-04-12T23:20:50.100Z +1985-04-12T23:20:50Z +1985-04-12T23:20:50.0Z +1985-04-12T23:20:50.123+00:00 +1985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123+07:00 +1985-04-12T23:20:50.123+01:45 +0985-04-12T23:20:50.123-07:00 +1985-04-12T23:20:50.123-07:00 +0123-01-01T00:00:00.000Z -- 2.51.2 From cd14204ba768a7f04432f45156828e2bc06b8455 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 1 Oct 2023 16:37:24 -0700 Subject: [PATCH 168/192] syntax: basic language code (BCP-47) validation --- atproto/syntax/language.go | 41 +++++++++++++++ atproto/syntax/language_test.go | 50 +++++++++++++++++++ .../testdata/language_syntax_invalid.txt | 10 ++++ .../syntax/testdata/language_syntax_valid.txt | 18 +++++++ 4 files changed, 119 insertions(+) create mode 100644 atproto/syntax/language.go create mode 100644 atproto/syntax/language_test.go create mode 100644 atproto/syntax/testdata/language_syntax_invalid.txt create mode 100644 atproto/syntax/testdata/language_syntax_valid.txt diff --git a/atproto/syntax/language.go b/atproto/syntax/language.go new file mode 100644 index 00000000..550801f2 --- /dev/null +++ b/atproto/syntax/language.go @@ -0,0 +1,41 @@ +package syntax + +import ( + "fmt" + "regexp" +) + +// Represents a Language specifier in string format, as would pass Lexicon syntax validation. +// +// Always use [ParseLanguage] instead of wrapping strings directly, especially when working with network input. +// +// The syntax is BCP-47. This is a partial/naive parsing implementation, designed for fast validation and exact-string passthrough with no normaliztion. For actually working with BCP-47 language specifiers in atproto code bases, we recommend the golang.org/x/text/language package. +type Language string + +func ParseLanguage(raw string) (Language, error) { + if len(raw) > 128 { + return "", fmt.Errorf("Language is too long (128 chars max)") + } + var langRegex = regexp.MustCompile(`^(i|[a-z]{2,3})(-[a-zA-Z0-9]+)*$`) + if !langRegex.MatchString(raw) { + return "", fmt.Errorf("Language syntax didn't validate via regex") + } + return Language(raw), nil +} + +func (l Language) String() string { + return string(l) +} + +func (l Language) MarshalText() ([]byte, error) { + return []byte(l.String()), nil +} + +func (l *Language) UnmarshalText(text []byte) error { + lang, err := ParseLanguage(string(text)) + if err != nil { + return err + } + *l = lang + return nil +} diff --git a/atproto/syntax/language_test.go b/atproto/syntax/language_test.go new file mode 100644 index 00000000..96556b0a --- /dev/null +++ b/atproto/syntax/language_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropLanguagesValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/language_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseLanguage(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropLanguagesInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/language_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseLanguage(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} diff --git a/atproto/syntax/testdata/language_syntax_invalid.txt b/atproto/syntax/testdata/language_syntax_invalid.txt new file mode 100644 index 00000000..9bba437a --- /dev/null +++ b/atproto/syntax/testdata/language_syntax_invalid.txt @@ -0,0 +1,10 @@ +jaja +. +123 +JA +j +ja- +a-DE + +# technically not valid, but allowing in naive parser +#de-419-DE diff --git a/atproto/syntax/testdata/language_syntax_valid.txt b/atproto/syntax/testdata/language_syntax_valid.txt new file mode 100644 index 00000000..8a4a5fee --- /dev/null +++ b/atproto/syntax/testdata/language_syntax_valid.txt @@ -0,0 +1,18 @@ +ja +ban +pt-BR +hy-Latn-IT-arevela +en-GB +zh-Hant +sgn-BE-NL +es-419 +en-GB-boont-r-extended-sequence-x-private + +# grandfathered +zh-hakka +i-default +i-navajo + +# https://github.com/sebinsua/ietf-language-tag-regex/blob/master/test.js +de-CH-1901 +qaa-Qaaa-QM-x-southern -- 2.51.2 From 55a81941398b3a264384543509aa6434a1ddb9f4 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 1 Oct 2023 16:37:50 -0700 Subject: [PATCH 169/192] syntax: basic/generic/permissive URI validation --- .../syntax/testdata/uri_syntax_invalid.txt | 17 +++++++ atproto/syntax/testdata/uri_syntax_valid.txt | 14 ++++++ atproto/syntax/uri.go | 41 +++++++++++++++ atproto/syntax/uri_test.go | 50 +++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 atproto/syntax/testdata/uri_syntax_invalid.txt create mode 100644 atproto/syntax/testdata/uri_syntax_valid.txt create mode 100644 atproto/syntax/uri.go create mode 100644 atproto/syntax/uri_test.go diff --git a/atproto/syntax/testdata/uri_syntax_invalid.txt b/atproto/syntax/testdata/uri_syntax_invalid.txt new file mode 100644 index 00000000..f79a4ba1 --- /dev/null +++ b/atproto/syntax/testdata/uri_syntax_invalid.txt @@ -0,0 +1,17 @@ + +example.com +://example.com +//example.com +http: +.http://example.com +-http://example.com +12345 +127.0.0.1 + +https://example.com/path gap + https://example.com/path +https://example.com/trailing-whitespace + +# too long (max 8 kbytes) +# python: "https://example.com/" + 8200 *"x" +https://example.com/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx diff --git a/atproto/syntax/testdata/uri_syntax_valid.txt b/atproto/syntax/testdata/uri_syntax_valid.txt new file mode 100644 index 00000000..709a6136 --- /dev/null +++ b/atproto/syntax/testdata/uri_syntax_valid.txt @@ -0,0 +1,14 @@ + +https://example.com +https://example.com/path?q=blah&yes=true#frag.123 +dns:example.com +at://handle.example.com/nsid/rkey +did:key:zQ3shZc2QzApp2oymGvQbzP8eKheVshBHbU4ZYjeXqwSKEn6N +content-type:text/plan +microsoft.windows.camera:thing +go://?Mercedes%20Benz + +# long (but not too long) +# python: "https://example.com/" + 5000*"x" +https://example.com/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx + diff --git a/atproto/syntax/uri.go b/atproto/syntax/uri.go new file mode 100644 index 00000000..646fd38d --- /dev/null +++ b/atproto/syntax/uri.go @@ -0,0 +1,41 @@ +package syntax + +import ( + "fmt" + "regexp" +) + +// Represents an arbitary URI in string format, as would pass Lexicon syntax validation. +// +// The syntax is minimal and permissive, designed for fast verification and exact-string passthrough, not schema-specific parsing or validation. For example, will not validate AT-URI or DID strings. +// +// Always use [ParseURI] instead of wrapping strings directly, especially when working with network input. +type URI string + +func ParseURI(raw string) (URI, error) { + if len(raw) > 8192 { + return "", fmt.Errorf("URI is too long (8192 chars max)") + } + var uriRegex = regexp.MustCompile(`^[a-z][a-z.-]{0,80}:[[:graph:]]+$`) + if !uriRegex.MatchString(raw) { + return "", fmt.Errorf("URI syntax didn't validate via regex") + } + return URI(raw), nil +} + +func (u URI) String() string { + return string(u) +} + +func (u URI) MarshalText() ([]byte, error) { + return []byte(u.String()), nil +} + +func (u *URI) UnmarshalText(text []byte) error { + uri, err := ParseURI(string(text)) + if err != nil { + return err + } + *u = uri + return nil +} diff --git a/atproto/syntax/uri_test.go b/atproto/syntax/uri_test.go new file mode 100644 index 00000000..cca4a6d7 --- /dev/null +++ b/atproto/syntax/uri_test.go @@ -0,0 +1,50 @@ +package syntax + +import ( + "bufio" + "fmt" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestInteropURIsValid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/uri_syntax_valid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseURI(line) + if err != nil { + fmt.Println("GOOD: " + line) + } + assert.NoError(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropURIsInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/uri_syntax_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseURI(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} -- 2.51.2 From ba4dae03a1be1d027542638b1f73a769a83b0049 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Sun, 1 Oct 2023 17:07:52 -0700 Subject: [PATCH 170/192] syntax: datetime helpers --- atproto/syntax/datetime.go | 31 +++++++++++++++++ atproto/syntax/datetime_test.go | 34 +++++++++++++++++-- .../testdata/datetime_parse_invalid.txt | 7 ++++ .../testdata/datetime_syntax_invalid.txt | 8 ----- .../syntax/testdata/datetime_syntax_valid.txt | 14 ++++++++ 5 files changed, 83 insertions(+), 11 deletions(-) create mode 100644 atproto/syntax/testdata/datetime_parse_invalid.txt diff --git a/atproto/syntax/datetime.go b/atproto/syntax/datetime.go index 28279cb1..36d4419b 100644 --- a/atproto/syntax/datetime.go +++ b/atproto/syntax/datetime.go @@ -4,6 +4,14 @@ import ( "fmt" "regexp" "strings" + "time" +) + +const ( + // Prefered atproto Datetime string syntax, for use with [time.Format]. + // + // Note that *parsing* syntax is more flexible. + AtprotoDatetimeLayout = "2006-01-02T15:04:05.999Z" ) // Represents the a Datetime in string format, as would pass Lexicon syntax validation: the intersection of RFC-3339 and ISO-8601 syntax. @@ -25,6 +33,29 @@ func ParseDatetime(raw string) (Datetime, error) { return Datetime(raw), nil } +// Parses a string to a golang time.Time in a single step. +func ParseDatetimeTime(raw string) (time.Time, error) { + var zero time.Time + d, err := ParseDatetime(raw) + if err != nil { + return zero, err + } + return d.Time() +} + +// Parses the Datetime string in to a golang time.Time. +// +// There are a small number of strings which will pass initial syntax validation but fail when actually parsing, so this function can return an error. Use [ParseDatetimeTime] to fully parse in a single function call. +func (d Datetime) Time() (time.Time, error) { + return time.Parse(time.RFC3339Nano, d.String()) +} + +// Creates a new valid Datetime string matching the current time, in prefered syntax. +func DatetimeNow() Datetime { + t := time.Now().UTC() + return Datetime(t.Format(AtprotoDatetimeLayout)) +} + func (d Datetime) String() string { return string(d) } diff --git a/atproto/syntax/datetime_test.go b/atproto/syntax/datetime_test.go index 0e93cb17..a47fb7c1 100644 --- a/atproto/syntax/datetime_test.go +++ b/atproto/syntax/datetime_test.go @@ -9,7 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestInteropDatetimesValid(t *testing.T) { +func TestInteropDatetimeValid(t *testing.T) { assert := assert.New(t) file, err := os.Open("testdata/datetime_syntax_valid.txt") assert.NoError(err) @@ -20,7 +20,7 @@ func TestInteropDatetimesValid(t *testing.T) { if len(line) == 0 || line[0] == '#' { continue } - _, err := ParseDatetime(line) + _, err := ParseDatetimeTime(line) if err != nil { fmt.Println("GOOD: " + line) } @@ -29,7 +29,7 @@ func TestInteropDatetimesValid(t *testing.T) { assert.NoError(scanner.Err()) } -func TestInteropDatetimesInvalid(t *testing.T) { +func TestInteropDatetimeInvalid(t *testing.T) { assert := assert.New(t) file, err := os.Open("testdata/datetime_syntax_invalid.txt") assert.NoError(err) @@ -48,3 +48,31 @@ func TestInteropDatetimesInvalid(t *testing.T) { } assert.NoError(scanner.Err()) } + +func TestInteropDatetimeTimeInvalid(t *testing.T) { + assert := assert.New(t) + file, err := os.Open("testdata/datetime_parse_invalid.txt") + assert.NoError(err) + defer file.Close() + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 || line[0] == '#' { + continue + } + _, err := ParseDatetimeTime(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + } + assert.NoError(scanner.Err()) +} + +func TestInteropDatetimeNow(t *testing.T) { + assert := assert.New(t) + + dt := DatetimeNow() + _, err := ParseDatetimeTime(dt.String()) + assert.NoError(err) +} diff --git a/atproto/syntax/testdata/datetime_parse_invalid.txt b/atproto/syntax/testdata/datetime_parse_invalid.txt new file mode 100644 index 00000000..1ab8f6d0 --- /dev/null +++ b/atproto/syntax/testdata/datetime_parse_invalid.txt @@ -0,0 +1,7 @@ +# superficial syntax parses ok, but are not valid datetimes for semantic reasons (eg, "month zero") +#1985-00-12T23:20:50.123Z +#1985-04-00T23:20:50.123Z +#1985-13-12T23:20:50.123Z +#1985-04-12T25:20:50.123Z +#1985-04-12T23:61:50.123Z +#1985-04-12T23:20:61.123Z diff --git a/atproto/syntax/testdata/datetime_syntax_invalid.txt b/atproto/syntax/testdata/datetime_syntax_invalid.txt index c4a0e764..a686f717 100644 --- a/atproto/syntax/testdata/datetime_syntax_invalid.txt +++ b/atproto/syntax/testdata/datetime_syntax_invalid.txt @@ -16,14 +16,6 @@ 1985-04-12T23:20:50.123Z 1985-04-12T 23:20:50.123Z -# TODO: full parse to validate? -#1985-00-12T23:20:50.123Z -#1985-04-00T23:20:50.123Z -#1985-13-12T23:20:50.123Z -#1985-04-12T25:20:50.123Z -#1985-04-12T23:61:50.123Z -#1985-04-12T23:20:61.123Z - # not enough zero padding 1985-4-12T23:20:50.123Z 1985-04-2T23:20:50.123Z diff --git a/atproto/syntax/testdata/datetime_syntax_valid.txt b/atproto/syntax/testdata/datetime_syntax_valid.txt index 798c7485..79ed5cab 100644 --- a/atproto/syntax/testdata/datetime_syntax_valid.txt +++ b/atproto/syntax/testdata/datetime_syntax_valid.txt @@ -18,3 +18,17 @@ 0985-04-12T23:20:50.123-07:00 1985-04-12T23:20:50.123-07:00 0123-01-01T00:00:00.000Z + +# various precisions, up through at least 12 digits +1985-04-12T23:20:50.1Z +1985-04-12T23:20:50.12Z +1985-04-12T23:20:50.123Z +1985-04-12T23:20:50.1234Z +1985-04-12T23:20:50.12345Z +1985-04-12T23:20:50.123456Z +1985-04-12T23:20:50.1234567Z +1985-04-12T23:20:50.12345678Z +1985-04-12T23:20:50.123456789Z +1985-04-12T23:20:50.1234567890Z +1985-04-12T23:20:50.12345678901Z +1985-04-12T23:20:50.123456789012Z -- 2.51.2 From 28897c5b8f424010a4a43f956c6c7bf125bac8a4 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 2 Oct 2023 00:00:50 -0700 Subject: [PATCH 171/192] syntax: TID helpers, including generator 'clock' --- atproto/syntax/tid.go | 91 +++++++++++++++++++++++++++++++++++++- atproto/syntax/tid_test.go | 77 ++++++++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 1 deletion(-) diff --git a/atproto/syntax/tid.go b/atproto/syntax/tid.go index 47f3ebda..00176974 100644 --- a/atproto/syntax/tid.go +++ b/atproto/syntax/tid.go @@ -1,10 +1,22 @@ package syntax import ( + "encoding/base32" "fmt" "regexp" + "strings" + "sync" + "time" ) +const ( + Base32SortAlphabet = "234567abcdefghijklmnopqrstuvwxyz" +) + +func Base32Sort() *base32.Encoding { + return base32.NewEncoding(Base32SortAlphabet).WithPadding(base32.NoPadding) +} + // Represents a TID in string format, as would pass Lexicon syntax validation. // // Always use [ParseTID] instead of wrapping strings directly, especially when working with network input. @@ -23,7 +35,58 @@ func ParseTID(raw string) (TID, error) { return TID(raw), nil } -// TODO: additional helpers: to timestamp, from timestamp, from integer, etc +// Naive (unsafe) one-off TID generation with the current time. +// +// You should usually use a [TIDClock] to ensure monotonic output. +func NewTIDNow(clockId uint) TID { + return NewTID(time.Now().UTC().UnixMicro(), clockId) +} + +func NewTIDFromInteger(v uint64) TID { + v = (0x7FFF_FFFF_FFFF_FFFF & v) + s := "" + for i := 0; i < 13; i++ { + s = string(Base32SortAlphabet[v&0x1F]) + s + v = v >> 5 + } + return TID(s) +} + +// Constructs a new TID from a UNIX timestamp (in milliseconds) and clock ID value. +func NewTID(unixMilis int64, clockId uint) TID { + var v uint64 = (uint64(unixMilis&0x1F_FFFF_FFFF_FFFF) << 10) | uint64(clockId&0x3FF) + return NewTIDFromInteger(v) +} + +// Returns full integer representation of this TID (not used often) +func (t TID) Integer() uint64 { + s := t.String() + if len(s) != 13 { + return 0 + } + var v uint64 + for i := 0; i < 13; i++ { + c := strings.IndexByte(Base32SortAlphabet, s[i]) + if c < 0 { + return 0 + } + v = (v << 5) | uint64(c&0x1F) + } + return v +} + +// Returns the golang [time.Time] corresponding to this TID's timestamp. +func (t TID) Time() time.Time { + i := t.Integer() + i = (i >> 10) & 0x1FFF_FFFF_FFFF_FFFF + return time.UnixMicro(int64(i)).UTC() +} + +// Returns the clock ID part of this TID, as an unsigned integer +func (t TID) ClockID() uint { + i := t.Integer() + return uint(i & 0x3FF) +} func (t TID) String() string { return string(t) @@ -41,3 +104,29 @@ func (t *TID) UnmarshalText(text []byte) error { *t = tid return nil } + +// TID generator, which keeps state to ensure TID values always monotonically increase. +// +// Uses [sync.Mutex], so may block briefly but safe for concurrent use. +type TIDClock struct { + ClockID uint + mtx sync.Mutex + lastUnixMicro int64 +} + +func NewTIDClock(clockId uint) *TIDClock { + return &TIDClock{ + ClockID: clockId, + } +} + +func (c *TIDClock) Next() TID { + now := time.Now().UTC().UnixMicro() + c.mtx.Lock() + if now <= c.lastUnixMicro { + now = c.lastUnixMicro + 1 + } + c.lastUnixMicro = now + c.mtx.Unlock() + return NewTID(now, c.ClockID) +} diff --git a/atproto/syntax/tid_test.go b/atproto/syntax/tid_test.go index 9d2c883a..9a7b61e8 100644 --- a/atproto/syntax/tid_test.go +++ b/atproto/syntax/tid_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -48,3 +49,79 @@ func TestInteropTIDsInvalid(t *testing.T) { } assert.NoError(scanner.Err()) } + +func TestTIDParts(t *testing.T) { + assert := assert.New(t) + + raw := "3kao2cl6lyj2p" + tid, err := ParseTID(raw) + assert.NoError(err) + // TODO: assert.Equal(uint64(0x181a8044491f3bec), tid.Integer()) + // TODO: assert.Equal(uint(1004), tid.ClockID()) + assert.Equal(2023, tid.Time().Year()) + + out := NewTID(tid.Time().UnixMicro(), tid.ClockID()) + assert.Equal(raw, out.String()) + assert.Equal(tid.ClockID(), out.ClockID()) + assert.Equal(tid.Time(), out.Time()) + assert.Equal(tid.Integer(), out.Integer()) + + out2 := NewTIDFromInteger(tid.Integer()) + assert.Equal(tid.Integer(), out2.Integer()) +} + +func TestTIDExamples(t *testing.T) { + assert := assert.New(t) + // TODO: seems like TS code might be wrong? "242k52k4kg3s2" + assert.Equal("242k52k4kg3sc", NewTIDFromInteger(0x0102030405060708).String()) + assert.Equal(uint64(0x0102030405060708), TID("242k52k4kg3sc").Integer()) + //assert.Equal("2222222222222", NewTIDFromInteger(0x0000000000000000).String()) + //assert.Equal(uint64(), TID("242k52k4kg3s2").Integer()) + assert.Equal("2222222222223", NewTIDFromInteger(0x0000000000000001).String()) + assert.Equal(uint64(0x0000000000000001), TID("2222222222223").Integer()) + + assert.Equal("6222222222222", NewTIDFromInteger(0x4000000000000000).String()) + assert.Equal(uint64(0x4000000000000000), TID("6222222222222").Integer()) + + // ignoring type byte + assert.Equal("2222222222222", NewTIDFromInteger(0x8000000000000000).String()) +} + +func TestTIDNoPanic(t *testing.T) { + for _, s := range []string{"", "3jzfcijpj2z2aa", "3jzfcijpj2z2", ".."} { + bad := TID(s) + _ = bad.ClockID() + _ = bad.Integer() + _ = bad.Time() + _ = bad.String() + } +} + +func TestTIDConstruction(t *testing.T) { + assert := assert.New(t) + + zero := NewTID(0, 0) + assert.Equal("2222222222222", zero.String()) + assert.Equal(uint64(0), zero.Integer()) + assert.Equal(uint(0), zero.ClockID()) + assert.Equal(time.UnixMilli(0).UTC(), zero.Time()) + + now := NewTIDNow(1011) + assert.Equal(uint(1011), now.ClockID()) + assert.True(time.Since(now.Time()) < time.Minute) + + over := NewTIDNow(4096) + assert.Equal(uint(0), over.ClockID()) +} + +func TestTIDClock(t *testing.T) { + assert := assert.New(t) + + clk := NewTIDClock(0) + last := NewTID(0, 0) + for i := 0; i < 100; i++ { + next := clk.Next() + assert.Greater(next, last) + last = next + } +} -- 2.51.2 From b80a1e1bdad8031daac53f7a083751a8e2ebca81 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Mon, 2 Oct 2023 23:10:24 -0700 Subject: [PATCH 172/192] syntax: always do full datetime parse --- atproto/syntax/datetime.go | 26 +++++++++++++++++++------- atproto/syntax/datetime_test.go | 9 +++++++-- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/atproto/syntax/datetime.go b/atproto/syntax/datetime.go index 36d4419b..7910d6e3 100644 --- a/atproto/syntax/datetime.go +++ b/atproto/syntax/datetime.go @@ -17,6 +17,8 @@ const ( // Represents the a Datetime in string format, as would pass Lexicon syntax validation: the intersection of RFC-3339 and ISO-8601 syntax. // // Always use [ParseDatetime] instead of wrapping strings directly, especially when working with network input. +// +// Syntax is specified at: https://atproto.com/specs/lexicon#datetime type Datetime string func ParseDatetime(raw string) (Datetime, error) { @@ -30,24 +32,34 @@ func ParseDatetime(raw string) (Datetime, error) { if strings.HasSuffix(raw, "-00:00") { return "", fmt.Errorf("Datetime can't use '-00:00' for UTC timezone, must use '+00:00', per ISO-8601") } + // ensure that the datetime actually parses using golang time lib + _, err := time.Parse(time.RFC3339Nano, raw) + if err != nil { + return "", err + } return Datetime(raw), nil } -// Parses a string to a golang time.Time in a single step. +// Validates and converts a string to a golang [time.Time] in a single step. func ParseDatetimeTime(raw string) (time.Time, error) { - var zero time.Time d, err := ParseDatetime(raw) if err != nil { + var zero time.Time return zero, err } - return d.Time() + return d.Time(), nil } -// Parses the Datetime string in to a golang time.Time. +// Parses the Datetime string in to a golang [time.Time]. // -// There are a small number of strings which will pass initial syntax validation but fail when actually parsing, so this function can return an error. Use [ParseDatetimeTime] to fully parse in a single function call. -func (d Datetime) Time() (time.Time, error) { - return time.Parse(time.RFC3339Nano, d.String()) +// This method assumes that [ParseDatetime] was used to create the Datetime, which already verified parsing, and thus that [time.Parse] will always succeed. In the event of an error, zero/nil will be returned. +func (d Datetime) Time() time.Time { + var zero time.Time + ret, err := time.Parse(time.RFC3339Nano, d.String()) + if err != nil { + return zero + } + return ret } // Creates a new valid Datetime string matching the current time, in prefered syntax. diff --git a/atproto/syntax/datetime_test.go b/atproto/syntax/datetime_test.go index a47fb7c1..c12e20e8 100644 --- a/atproto/syntax/datetime_test.go +++ b/atproto/syntax/datetime_test.go @@ -60,7 +60,12 @@ func TestInteropDatetimeTimeInvalid(t *testing.T) { if len(line) == 0 || line[0] == '#' { continue } - _, err := ParseDatetimeTime(line) + _, err := ParseDatetime(line) + if err == nil { + fmt.Println("BAD: " + line) + } + assert.Error(err) + _, err = ParseDatetimeTime(line) if err == nil { fmt.Println("BAD: " + line) } @@ -69,7 +74,7 @@ func TestInteropDatetimeTimeInvalid(t *testing.T) { assert.NoError(scanner.Err()) } -func TestInteropDatetimeNow(t *testing.T) { +func TestDatetimeNow(t *testing.T) { assert := assert.New(t) dt := DatetimeNow() -- 2.51.2 From 66746fd5106fbdb650c606526aa0cc9726868133 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Tue, 3 Oct 2023 00:17:27 -0700 Subject: [PATCH 173/192] syntax: un-comment datetime syntax lines --- atproto/syntax/testdata/datetime_parse_invalid.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/atproto/syntax/testdata/datetime_parse_invalid.txt b/atproto/syntax/testdata/datetime_parse_invalid.txt index 1ab8f6d0..3672453a 100644 --- a/atproto/syntax/testdata/datetime_parse_invalid.txt +++ b/atproto/syntax/testdata/datetime_parse_invalid.txt @@ -1,7 +1,7 @@ # superficial syntax parses ok, but are not valid datetimes for semantic reasons (eg, "month zero") -#1985-00-12T23:20:50.123Z -#1985-04-00T23:20:50.123Z -#1985-13-12T23:20:50.123Z -#1985-04-12T25:20:50.123Z -#1985-04-12T23:61:50.123Z -#1985-04-12T23:20:61.123Z +1985-00-12T23:20:50.123Z +1985-04-00T23:20:50.123Z +1985-13-12T23:20:50.123Z +1985-04-12T25:20:50.123Z +1985-04-12T23:99:50.123Z +1985-04-12T23:20:61.123Z -- 2.51.2 From a135de9a2444f088002ba5bbb783211b926995f2 Mon Sep 17 00:00:00 2001 From: bryan newbold Date: Wed, 4 Oct 2023 00:18:39 -0700 Subject: [PATCH 174/192] syntax: fix TID milli/micro; add NewTIDFromTime --- atproto/syntax/tid.go | 9 +++++++-- atproto/syntax/tid_test.go | 6 +++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/atproto/syntax/tid.go b/atproto/syntax/tid.go index 00176974..5bed77d5 100644 --- a/atproto/syntax/tid.go +++ b/atproto/syntax/tid.go @@ -53,11 +53,16 @@ func NewTIDFromInteger(v uint64) TID { } // Constructs a new TID from a UNIX timestamp (in milliseconds) and clock ID value. -func NewTID(unixMilis int64, clockId uint) TID { - var v uint64 = (uint64(unixMilis&0x1F_FFFF_FFFF_FFFF) << 10) | uint64(clockId&0x3FF) +func NewTID(unixMicros int64, clockId uint) TID { + var v uint64 = (uint64(unixMicros&0x1F_FFFF_FFFF_FFFF) << 10) | uint64(clockId&0x3FF) return NewTIDFromInteger(v) } +// Constructs a new TID from a [time.Time] and clock ID value +func NewTIDFromTime(ts time.Time, clockId uint) TID { + return NewTID(ts.UTC().UnixMicro(), clockId) +} + // Returns full integer representation of this TID (not used often) func (t TID) Integer() uint64 { s := t.String() diff --git a/atproto/syntax/tid_test.go b/atproto/syntax/tid_test.go index 9a7b61e8..ed0d94be 100644 --- a/atproto/syntax/tid_test.go +++ b/atproto/syntax/tid_test.go @@ -104,7 +104,7 @@ func TestTIDConstruction(t *testing.T) { assert.Equal("2222222222222", zero.String()) assert.Equal(uint64(0), zero.Integer()) assert.Equal(uint(0), zero.ClockID()) - assert.Equal(time.UnixMilli(0).UTC(), zero.Time()) + assert.Equal(time.UnixMicro(0).UTC(), zero.Time()) now := NewTIDNow(1011) assert.Equal(uint(1011), now.ClockID()) @@ -112,6 +112,10 @@ func TestTIDConstruction(t *testing.T) { over := NewTIDNow(4096) assert.Equal(uint(0), over.ClockID()) + + next := NewTIDFromTime(time.Now(), 123) + assert.Equal(uint(123), next.ClockID()) + assert.True(time.Since(next.Time()) < time.Minute) } func TestTIDClock(t *testing.T) { -- 2.51.2 From f5bc76235202800ee683b3f228d3d9b8c3f2f462 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 12 Oct 2023 06:48:54 +0000 Subject: [PATCH 175/192] Make BGS spidering optional --- cmd/bigsky/main.go | 7 ++++++- indexer/indexer.go | 11 +++++++---- indexer/posts_test.go | 2 +- pds/server.go | 2 +- testing/utils.go | 2 +- 5 files changed, 16 insertions(+), 8 deletions(-) diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index b11aa6d8..fd93c676 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -100,6 +100,11 @@ func run(args []string) { Name: "aggregation", Value: false, }, + &cli.BoolFlag{ + Name: "spidering", + Value: true, + EnvVars: []string{"BGS_SPIDERING"}, + }, &cli.StringFlag{ Name: "api-listen", Value: ":2470", @@ -278,7 +283,7 @@ func Bigsky(cctx *cli.Context) error { notifman := ¬ifs.NullNotifs{} - ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, repoman, true, cctx.Bool("aggregation")) + ix, err := indexer.NewIndexer(db, notifman, evtman, cachedidr, repoman, true, cctx.Bool("spidering"), cctx.Bool("aggregation")) if err != nil { return err } diff --git a/indexer/indexer.go b/indexer/indexer.go index 9a89c9cc..cdfcd191 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -51,13 +51,14 @@ type Indexer struct { LimitMux sync.RWMutex doAggregations bool + doSpider bool SendRemoteFollow func(context.Context, string, uint) error CreateExternalUser func(context.Context, string) (*models.ActorInfo, error) ApplyPDSClientSettings func(*xrpc.Client) } -func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, repoman *repomgr.RepoManager, crawl, aggregate bool) (*Indexer, error) { +func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events.EventManager, didr did.Resolver, repoman *repomgr.RepoManager, crawl, aggregate, spider bool) (*Indexer, error) { db.AutoMigrate(&models.FeedPost{}) db.AutoMigrate(&models.ActorInfo{}) db.AutoMigrate(&models.FollowRecord{}) @@ -72,6 +73,7 @@ func NewIndexer(db *gorm.DB, notifman notifs.NotificationManager, evtman *events didr: didr, Limiters: make(map[uint]*rate.Limiter), doAggregations: aggregate, + doSpider: spider, SendRemoteFollow: func(context.Context, string, uint) error { return nil }, @@ -181,10 +183,11 @@ func (ix *Indexer) handleRepoOp(ctx context.Context, evt *repomgr.RepoEvent, op return fmt.Errorf("handle recordCreate: %w", err) } } - if err := ix.crawlRecordReferences(ctx, op); err != nil { - return err + if ix.doSpider { + if err := ix.crawlRecordReferences(ctx, op); err != nil { + return err + } } - case repomgr.EvtKindDeleteRecord: if ix.doAggregations { if err := ix.handleRecordDelete(ctx, evt, op, true); err != nil { diff --git a/indexer/posts_test.go b/indexer/posts_test.go index ec48836b..cf79ff9c 100644 --- a/indexer/posts_test.go +++ b/indexer/posts_test.go @@ -61,7 +61,7 @@ func testIndexer(t *testing.T) *testIx { didr := testPLC(t) - ix, err := NewIndexer(maindb, notifman, evtman, didr, repoman, false, true) + ix, err := NewIndexer(maindb, notifman, evtman, didr, repoman, false, true, true) if err != nil { t.Fatal(err) } diff --git a/pds/server.go b/pds/server.go index 272e0e32..19d149bb 100644 --- a/pds/server.go +++ b/pds/server.go @@ -80,7 +80,7 @@ func NewServer(db *gorm.DB, cs *carstore.CarStore, serkey *did.PrivKey, handleSu repoman := repomgr.NewRepoManager(cs, kmgr) notifman := notifs.NewNotificationManager(db, repoman.GetRecord) - ix, err := indexer.NewIndexer(db, notifman, evtman, didr, repoman, false, true) + ix, err := indexer.NewIndexer(db, notifman, evtman, didr, repoman, false, true, true) if err != nil { return nil, err } diff --git a/testing/utils.go b/testing/utils.go index 9a216fe4..10a8467b 100644 --- a/testing/utils.go +++ b/testing/utils.go @@ -436,7 +436,7 @@ func SetupBGS(ctx context.Context, didr plc.PLCClient) (*TestBGS, error) { evtman := events.NewEventManager(diskpersist) - ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, repoman, true, true) + ix, err := indexer.NewIndexer(maindb, notifman, evtman, didr, repoman, true, true, true) if err != nil { return nil, err } -- 2.51.2 From 00f1a495c8c4c01588714af7240f4504355cc881 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 12 Oct 2023 18:39:14 +0000 Subject: [PATCH 176/192] Wire up compactor into BGS, better tracing, requeue routine --- bgs/bgs.go | 6 ++ bgs/compact.go | 178 +++++++++++++++++++++++++++------------------ bgs/metrics.go | 5 ++ cmd/bigsky/main.go | 4 +- 4 files changed, 120 insertions(+), 73 deletions(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 1199a1ae..11cf2fc3 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -143,6 +143,10 @@ func NewBGS(db *gorm.DB, ix *indexer.Indexer, repoman *repomgr.RepoManager, evtm return nil, err } + compactor := NewCompactor(nil) + compactor.Start(bgs) + bgs.compactor = compactor + return bgs, nil } @@ -366,6 +370,8 @@ func (bgs *BGS) Shutdown() []error { errs = append(errs, err) } + bgs.compactor.Shutdown() + return errs } diff --git a/bgs/compact.go b/bgs/compact.go index 4d0779ac..d200404b 100644 --- a/bgs/compact.go +++ b/bgs/compact.go @@ -33,6 +33,7 @@ func (q *queue) Append(uid models.Uid, fast bool) { q.q = append(q.q, queueItem{uid: uid, fast: fast}) q.members[uid] = struct{}{} + compactionQueueDepth.Inc() } func (q *queue) Prepend(uid models.Uid, fast bool) { @@ -45,6 +46,7 @@ func (q *queue) Prepend(uid models.Uid, fast bool) { q.q = append([]queueItem{{uid: uid, fast: fast}}, q.q...) q.members[uid] = struct{}{} + compactionQueueDepth.Inc() } func (q *queue) Has(uid models.Uid) bool { @@ -71,6 +73,7 @@ func (q *queue) Remove(uid models.Uid) { } delete(q.members, uid) + compactionQueueDepth.Dec() } func (q *queue) Pop() (*queueItem, bool) { @@ -85,6 +88,7 @@ func (q *queue) Pop() (*queueItem, bool) { q.q = q.q[1:] delete(q.members, item.uid) + compactionQueueDepth.Dec() return &item, true } @@ -96,19 +100,49 @@ type CompactorState struct { } type Compactor struct { - q *queue - state *CompactorState - stateLk sync.RWMutex - exit chan struct{} + q *queue + state *CompactorState + stateLk sync.RWMutex + exit chan struct{} + exited chan struct{} + requeueInterval time.Duration + requeueLimit int + requeueShardCount int + requeueFast bool } -func NewCompactor() *Compactor { +type CompactorOptions struct { + RequeueInterval time.Duration + RequeueLimit int + RequeueShardCount int + RequeueFast bool +} + +func DefaultCompactorOptions() *CompactorOptions { + return &CompactorOptions{ + RequeueInterval: time.Hour * 12, + RequeueLimit: 0, + RequeueShardCount: 50, + RequeueFast: true, + } +} + +func NewCompactor(opts *CompactorOptions) *Compactor { + if opts == nil { + opts = DefaultCompactorOptions() + } + return &Compactor{ q: &queue{ members: make(map[models.Uid]struct{}), }, - state: &CompactorState{}, - exit: make(chan struct{}), + state: &CompactorState{}, + exit: make(chan struct{}), + exited: make(chan struct{}), + requeueInterval: opts.RequeueInterval, + requeueLimit: opts.RequeueLimit, + requeueFast: opts.RequeueFast, + requeueShardCount: opts.RequeueShardCount, } } @@ -141,11 +175,55 @@ func (c *Compactor) GetState() *CompactorState { var errNoReposToCompact = fmt.Errorf("no repos to compact") -func (c *Compactor) Run(bgs *BGS) { +func (c *Compactor) Start(bgs *BGS) { + log.Info("starting compactor") + go c.DoWork(bgs) + go func() { + log.Infow("starting compactor requeue routine", + "interval", c.requeueInterval, + "limit", c.requeueLimit, + "shardCount", c.requeueShardCount, + "fast", c.requeueFast, + ) + + // Enqueue all repos on startup + ctx := context.Background() + ctx, span := otel.Tracer("compactor").Start(ctx, "RequeueRoutine") + if err := c.EnqueueAllRepos(ctx, bgs, c.requeueLimit, c.requeueShardCount, c.requeueFast); err != nil { + log.Errorw("failed to enqueue all repos", "err", err) + } + span.End() + + t := time.NewTicker(c.requeueInterval) + for { + select { + case <-c.exit: + return + case <-t.C: + ctx := context.Background() + ctx, span := otel.Tracer("compactor").Start(ctx, "RequeueRoutine") + if err := c.EnqueueAllRepos(ctx, bgs, c.requeueLimit, c.requeueShardCount, c.requeueFast); err != nil { + log.Errorw("failed to enqueue all repos", "err", err) + } + span.End() + } + } + }() +} + +func (c *Compactor) Shutdown() { + log.Info("stopping compactor") + close(c.exit) + <-c.exited + log.Info("compactor stopped") +} + +func (c *Compactor) DoWork(bgs *BGS) { for { select { case <-c.exit: - log.Warn("compactor exiting") + log.Info("compactor worker exiting, no more active compactions running") + close(c.exited) return default: } @@ -155,7 +233,7 @@ func (c *Compactor) Run(bgs *BGS) { state, err := c.CompactNext(ctx, bgs) if err != nil { if err == errNoReposToCompact { - log.Warn("no repos to compact, waiting and retrying") + log.Info("no repos to compact, waiting and retrying") time.Sleep(time.Second * 5) continue } @@ -170,7 +248,7 @@ func (c *Compactor) Run(bgs *BGS) { // Pause for a bit to avoid spamming failed compactions time.Sleep(time.Millisecond * 100) } else { - log.Warnw("compacted repo", + log.Infow("compacted repo", "uid", state.latestUID, "repo", state.latestDID, "status", state.status, @@ -182,7 +260,7 @@ func (c *Compactor) Run(bgs *BGS) { } func (c *Compactor) CompactNext(ctx context.Context, bgs *BGS) (*CompactorState, error) { - ctx, span := otel.Tracer("bgs").Start(ctx, "CompactNext") + ctx, span := otel.Tracer("compactor").Start(ctx, "CompactNext") defer span.End() item, ok := c.q.Pop() @@ -194,26 +272,40 @@ func (c *Compactor) CompactNext(ctx context.Context, bgs *BGS) (*CompactorState, user, err := bgs.lookupUserByUID(ctx, item.uid) if err != nil { + span.RecordError(err) c.SetState(item.uid, "unknown", "failed_getting_user", nil) return nil, fmt.Errorf("failed to get user %d: %w", item.uid, err) } + span.SetAttributes(attribute.String("repo", user.Did), attribute.Int("uid", int(item.uid))) + c.SetState(item.uid, user.Did, "compacting", nil) start := time.Now() st, err := bgs.repoman.CarStore().CompactUserShards(ctx, item.uid, item.fast) if err != nil { + span.RecordError(err) c.SetState(item.uid, user.Did, "failed_compacting", nil) return nil, fmt.Errorf("failed to compact shards for user %d: %w", item.uid, err) } compactionDuration.Observe(time.Since(start).Seconds()) + span.SetAttributes( + attribute.Int("shards.deleted", st.ShardsDeleted), + attribute.Int("shards.new", st.NewShards), + attribute.Int("dupes", st.DupeCount), + attribute.Int("shards.skipped", st.SkippedShards), + attribute.Int("refs", st.TotalRefs), + ) + c.SetState(item.uid, user.Did, "done", st) return c.GetState(), nil } func (c *Compactor) EnqueueRepo(ctx context.Context, user User, fast bool) { + ctx, span := otel.Tracer("compactor").Start(ctx, "EnqueueRepo") + defer span.End() log.Infow("enqueueing compaction for repo", "repo", user.Did, "uid", user.ID, "fast", fast) c.q.Append(user.ID, fast) } @@ -223,7 +315,7 @@ func (c *Compactor) EnqueueRepo(ctx context.Context, user User, fast bool) { // shardCount is the number of shards to compact per user (0 = default of 50) // fast is whether to use the fast compaction method (skip large shards) func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shardCount int, fast bool) error { - ctx, span := otel.Tracer("bgs").Start(ctx, "EnqueueAllRepos") + ctx, span := otel.Tracer("compactor").Start(ctx, "EnqueueAllRepos") defer span.End() span.SetAttributes( @@ -239,7 +331,7 @@ func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shar span.SetAttributes(attribute.Int("clampedShardCount", shardCount)) log := log.With("source", "compactor_enqueue_all_repos", "lim", lim, "shardCount", shardCount, "fast", fast) - log.Warn("enqueueing all repos") + log.Info("enqueueing all repos") repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, shardCount) if err != nil { @@ -258,63 +350,7 @@ func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shar c.q.Append(r.Usr, fast) } - log.Warn("done enqueueing all repos") + log.Info("done enqueueing all repos") return nil } - -func (bgs *BGS) runRepoCompaction(ctx context.Context, lim int, dry bool, fast bool) (*compactionStats, error) { - ctx, span := otel.Tracer("bgs").Start(ctx, "runRepoCompaction") - defer span.End() - - log.Warn("starting repo compaction") - - runStart := time.Now() - - repos, err := bgs.repoman.CarStore().GetCompactionTargets(ctx, 50) - if err != nil { - return nil, fmt.Errorf("failed to get repos to compact: %w", err) - } - - if lim > 0 && len(repos) > lim { - repos = repos[:lim] - } - - if dry { - return &compactionStats{ - Targets: repos, - }, nil - } - - results := make(map[models.Uid]*carstore.CompactionStats) - for i, r := range repos { - select { - case <-ctx.Done(): - return &compactionStats{ - Targets: repos, - Completed: results, - }, nil - default: - } - - repostart := time.Now() - st, err := bgs.repoman.CarStore().CompactUserShards(context.Background(), r.Usr, fast) - if err != nil { - log.Errorf("failed to compact shards for user %d: %s", r.Usr, err) - continue - } - compactionDuration.Observe(time.Since(repostart).Seconds()) - results[r.Usr] = st - - if i%100 == 0 { - log.Warnf("compacted %d repos in %s", i+1, time.Since(runStart)) - } - } - - log.Warnf("compacted %d repos in %s", len(repos), time.Since(runStart)) - - return &compactionStats{ - Targets: repos, - Completed: results, - }, nil -} diff --git a/bgs/metrics.go b/bgs/metrics.go index 12b401e0..b33677e6 100644 --- a/bgs/metrics.go +++ b/bgs/metrics.go @@ -48,6 +48,11 @@ var compactionDuration = promauto.NewHistogram(prometheus.HistogramOpts{ Buckets: prometheus.ExponentialBuckets(0.001, 3, 14), }) +var compactionQueueDepth = promauto.NewGauge(prometheus.GaugeOpts{ + Name: "compaction_queue_depth", + Help: "The current depth of the compaction queue", +}) + var newUsersDiscovered = promauto.NewCounter(prometheus.CounterOpts{ Name: "bgs_new_users_discovered", Help: "The total number of new users discovered directly from the firehose (not from refs)", diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index fd93c676..e571628f 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -11,7 +11,7 @@ import ( "time" "github.com/bluesky-social/indigo/api" - "github.com/bluesky-social/indigo/bgs" + libbgs "github.com/bluesky-social/indigo/bgs" "github.com/bluesky-social/indigo/blobs" "github.com/bluesky-social/indigo/carstore" "github.com/bluesky-social/indigo/did" @@ -335,7 +335,7 @@ func Bigsky(cctx *cli.Context) error { } log.Infow("constructing bgs") - bgs, err := bgs.NewBGS(db, ix, repoman, evtman, cachedidr, blobstore, hr, !cctx.Bool("crawl-insecure-ws")) + bgs, err := libbgs.NewBGS(db, ix, repoman, evtman, cachedidr, blobstore, hr, !cctx.Bool("crawl-insecure-ws")) if err != nil { return err } -- 2.51.2 From 4c4f5c8acd43b1fe635c9518332074f2a371168b Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 12 Oct 2023 19:25:50 +0000 Subject: [PATCH 177/192] Rename some stuff and add a few comments --- bgs/compact.go | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/bgs/compact.go b/bgs/compact.go index d200404b..1d04c89d 100644 --- a/bgs/compact.go +++ b/bgs/compact.go @@ -17,13 +17,15 @@ type queueItem struct { fast bool } -type queue struct { +// uniQueue is a queue that only allows one instance of a given uid +type uniQueue struct { q []queueItem members map[models.Uid]struct{} lk sync.Mutex } -func (q *queue) Append(uid models.Uid, fast bool) { +// Append appends a uid to the end of the queue if it doesn't already exist +func (q *uniQueue) Append(uid models.Uid, fast bool) { q.lk.Lock() defer q.lk.Unlock() @@ -36,7 +38,8 @@ func (q *queue) Append(uid models.Uid, fast bool) { compactionQueueDepth.Inc() } -func (q *queue) Prepend(uid models.Uid, fast bool) { +// Prepend prepends a uid to the beginning of the queue if it doesn't already exist +func (q *uniQueue) Prepend(uid models.Uid, fast bool) { q.lk.Lock() defer q.lk.Unlock() @@ -49,7 +52,8 @@ func (q *queue) Prepend(uid models.Uid, fast bool) { compactionQueueDepth.Inc() } -func (q *queue) Has(uid models.Uid) bool { +// Has returns true if the queue contains the given uid +func (q *uniQueue) Has(uid models.Uid) bool { q.lk.Lock() defer q.lk.Unlock() @@ -57,7 +61,8 @@ func (q *queue) Has(uid models.Uid) bool { return ok } -func (q *queue) Remove(uid models.Uid) { +// Remove removes the given uid from the queue +func (q *uniQueue) Remove(uid models.Uid) { q.lk.Lock() defer q.lk.Unlock() @@ -76,7 +81,8 @@ func (q *queue) Remove(uid models.Uid) { compactionQueueDepth.Dec() } -func (q *queue) Pop() (*queueItem, bool) { +// Pop pops the first item off the front of the queue +func (q *uniQueue) Pop() (*queueItem, bool) { q.lk.Lock() defer q.lk.Unlock() @@ -99,8 +105,9 @@ type CompactorState struct { stats *carstore.CompactionStats } +// Compactor is a compactor daemon that compacts repos in the background type Compactor struct { - q *queue + q *uniQueue state *CompactorState stateLk sync.RWMutex exit chan struct{} @@ -133,7 +140,7 @@ func NewCompactor(opts *CompactorOptions) *Compactor { } return &Compactor{ - q: &queue{ + q: &uniQueue{ members: make(map[models.Uid]struct{}), }, state: &CompactorState{}, @@ -175,9 +182,10 @@ func (c *Compactor) GetState() *CompactorState { var errNoReposToCompact = fmt.Errorf("no repos to compact") +// Start starts the compactor func (c *Compactor) Start(bgs *BGS) { log.Info("starting compactor") - go c.DoWork(bgs) + go c.doWork(bgs) go func() { log.Infow("starting compactor requeue routine", "interval", c.requeueInterval, @@ -211,6 +219,7 @@ func (c *Compactor) Start(bgs *BGS) { }() } +// Shutdown shuts down the compactor func (c *Compactor) Shutdown() { log.Info("stopping compactor") close(c.exit) @@ -218,7 +227,7 @@ func (c *Compactor) Shutdown() { log.Info("compactor stopped") } -func (c *Compactor) DoWork(bgs *BGS) { +func (c *Compactor) doWork(bgs *BGS) { for { select { case <-c.exit: @@ -230,7 +239,7 @@ func (c *Compactor) DoWork(bgs *BGS) { ctx := context.Background() start := time.Now() - state, err := c.CompactNext(ctx, bgs) + state, err := c.compactNext(ctx, bgs) if err != nil { if err == errNoReposToCompact { log.Info("no repos to compact, waiting and retrying") @@ -259,7 +268,7 @@ func (c *Compactor) DoWork(bgs *BGS) { } } -func (c *Compactor) CompactNext(ctx context.Context, bgs *BGS) (*CompactorState, error) { +func (c *Compactor) compactNext(ctx context.Context, bgs *BGS) (*CompactorState, error) { ctx, span := otel.Tracer("compactor").Start(ctx, "CompactNext") defer span.End() -- 2.51.2 From ea4da6bc0c341c1d723b5584004b90868d5b2f75 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 12 Oct 2023 21:54:44 +0000 Subject: [PATCH 178/192] Logging update --- bgs/compact.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/compact.go b/bgs/compact.go index 1d04c89d..62e39b66 100644 --- a/bgs/compact.go +++ b/bgs/compact.go @@ -359,7 +359,7 @@ func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shar c.q.Append(r.Usr, fast) } - log.Info("done enqueueing all repos") + log.Infow("done enqueueing all repos", "repos_enqueued", len(repos)) return nil } -- 2.51.2 From 2de7856138c9ec9aa3819ab4b96bc33e593994f5 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Thu, 12 Oct 2023 22:44:02 +0000 Subject: [PATCH 179/192] Rename file --- bgs/{compact.go => compactor.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename bgs/{compact.go => compactor.go} (100%) diff --git a/bgs/compact.go b/bgs/compactor.go similarity index 100% rename from bgs/compact.go rename to bgs/compactor.go -- 2.51.2 From 58c3ca9c53536ab1dd82370ec87eca29fe85fc61 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 13 Oct 2023 01:03:49 +0000 Subject: [PATCH 180/192] Lower level of dirty dupes msg --- carstore/bs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/carstore/bs.go b/carstore/bs.go index 4b50e48d..e7ba0b79 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -1370,7 +1370,7 @@ func (cs *CarStore) CompactUserShards(ctx context.Context, user models.Uid, skip // still around but we're doing that anyways since compaction isnt a // perfect process - log.Warnw("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs)) + log.Debugw("repo has dirty dupes", "count", len(dupes), "uid", user, "staleRefs", len(staleRefs), "blockRefs", len(brefs)) //return nil, fmt.Errorf("WIP: not currently handling this case") } -- 2.51.2 From 3ec757d22ebff80f769fb196825985b78afcf374 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 13 Oct 2023 01:11:07 +0000 Subject: [PATCH 181/192] Make compactor less noisy when it has no work to do --- bgs/compactor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/compactor.go b/bgs/compactor.go index 62e39b66..77516a80 100644 --- a/bgs/compactor.go +++ b/bgs/compactor.go @@ -242,7 +242,7 @@ func (c *Compactor) doWork(bgs *BGS) { state, err := c.compactNext(ctx, bgs) if err != nil { if err == errNoReposToCompact { - log.Info("no repos to compact, waiting and retrying") + log.Debug("no repos to compact, waiting and retrying") time.Sleep(time.Second * 5) continue } -- 2.51.2 From 1fe2b2d22bb12ae4715d83470ac6e81cdd57f1eb Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 13 Oct 2023 01:13:13 +0000 Subject: [PATCH 182/192] Fix log levels for HTTP util --- util/http.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/http.go b/util/http.go index 3984b979..17bb8683 100644 --- a/util/http.go +++ b/util/http.go @@ -16,7 +16,7 @@ type LeveledZap struct { // re-writes HTTP client ERROR to WARN level (because of retries) func (l LeveledZap) Error(msg string, keysAndValues ...interface{}) { - l.inner.Warnw(msg, keysAndValues...) + l.inner.Errorw(msg, keysAndValues...) } func (l LeveledZap) Warn(msg string, keysAndValues ...interface{}) { @@ -29,7 +29,7 @@ func (l LeveledZap) Info(msg string, keysAndValues ...interface{}) { // re-writes HTTP client DEBUG to INFO level (this is where retry is logged) func (l LeveledZap) Debug(msg string, keysAndValues ...interface{}) { - l.inner.Infow(msg, keysAndValues...) + l.inner.Debugw(msg, keysAndValues...) } // Generates an HTTP client with decent general-purpose defaults around -- 2.51.2 From 7aa2eb18656ed10e87e58dbd3f31027f8cff281b Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 13 Oct 2023 01:13:51 +0000 Subject: [PATCH 183/192] Logging change on HTTP client --- util/http.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/util/http.go b/util/http.go index 17bb8683..00fadc2f 100644 --- a/util/http.go +++ b/util/http.go @@ -16,7 +16,7 @@ type LeveledZap struct { // re-writes HTTP client ERROR to WARN level (because of retries) func (l LeveledZap) Error(msg string, keysAndValues ...interface{}) { - l.inner.Errorw(msg, keysAndValues...) + l.inner.Warnw(msg, keysAndValues...) } func (l LeveledZap) Warn(msg string, keysAndValues ...interface{}) { @@ -27,7 +27,6 @@ func (l LeveledZap) Info(msg string, keysAndValues ...interface{}) { l.inner.Infow(msg, keysAndValues...) } -// re-writes HTTP client DEBUG to INFO level (this is where retry is logged) func (l LeveledZap) Debug(msg string, keysAndValues ...interface{}) { l.inner.Debugw(msg, keysAndValues...) } -- 2.51.2 From 34c826b4cfd4de4944db9ff08afe2b5a70e1a745 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 13 Oct 2023 01:15:25 +0000 Subject: [PATCH 184/192] Don't double-create new users --- bgs/bgs.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index 11cf2fc3..efa10e3a 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -808,7 +808,7 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event return fmt.Errorf("rebase was true in event seq:%d,host:%s", evt.Seq, host.Host) } - if host.ID != u.PDS { + if host.ID != u.PDS && u.PDS != 0 { log.Warnw("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host) subj, err := bgs.createExternalUser(ctx, evt.Repo) if err != nil { -- 2.51.2 From ae0818a9d2a1b7edac22324edd59fab5b2f7d1a3 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Fri, 13 Oct 2023 02:36:06 +0000 Subject: [PATCH 185/192] Update default compaction interval to 4 hours since it's really fast now --- bgs/compactor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/compactor.go b/bgs/compactor.go index 77516a80..a3cd5c49 100644 --- a/bgs/compactor.go +++ b/bgs/compactor.go @@ -127,7 +127,7 @@ type CompactorOptions struct { func DefaultCompactorOptions() *CompactorOptions { return &CompactorOptions{ - RequeueInterval: time.Hour * 12, + RequeueInterval: time.Hour * 4, RequeueLimit: 0, RequeueShardCount: 50, RequeueFast: true, -- 2.51.2 From 8cc02a27e3419a4d39ee52bc9bf462b40193abbf Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 12 Oct 2023 18:00:38 -0700 Subject: [PATCH 186/192] make it optional to hydrate records in internal events --- cmd/bigsky/main.go | 2 +- pds/server.go | 2 +- repomgr/repomgr.go | 150 +++++++++++++++++++++++++++------------------ testing/utils.go | 2 +- 4 files changed, 95 insertions(+), 61 deletions(-) diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index e571628f..391d3b26 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -307,7 +307,7 @@ func Bigsky(cctx *cli.Context) error { if err := ix.HandleRepoEvent(ctx, evt); err != nil { log.Errorw("failed to handle repo event", "err", err) } - }) + }, false) var blobstore blobs.BlobStore if bsdir := cctx.String("disk-blob-store"); bsdir != "" { diff --git a/pds/server.go b/pds/server.go index 19d149bb..8e51edcb 100644 --- a/pds/server.go +++ b/pds/server.go @@ -104,7 +104,7 @@ func NewServer(db *gorm.DB, cs *carstore.CarStore, serkey *did.PrivKey, handleSu if err := ix.HandleRepoEvent(ctx, evt); err != nil { log.Errorw("handle repo event failed", "user", evt.User, "err", err) } - }) + }, true) //ix.SendRemoteFollow = s.sendRemoteFollow ix.CreateExternalUser = s.createExternalUser diff --git a/repomgr/repomgr.go b/repomgr/repomgr.go index ba93bc75..f3c68137 100644 --- a/repomgr/repomgr.go +++ b/repomgr/repomgr.go @@ -45,8 +45,9 @@ type KeyManager interface { SignForUser(context.Context, string, []byte) ([]byte, error) } -func (rm *RepoManager) SetEventHandler(cb func(context.Context, *RepoEvent)) { +func (rm *RepoManager) SetEventHandler(cb func(context.Context, *RepoEvent), hydrateRecords bool) { rm.events = cb + rm.hydrateRecords = hydrateRecords } type RepoManager struct { @@ -56,7 +57,8 @@ type RepoManager struct { lklk sync.Mutex userLocks map[models.Uid]*userLock - events func(context.Context, *RepoEvent) + events func(context.Context, *RepoEvent) + hydrateRecords bool } type ActorInfo struct { @@ -250,19 +252,24 @@ func (rm *RepoManager) UpdateRecord(ctx context.Context, user models.Uid, collec } if rm.events != nil { + op := RepoOp{ + Kind: EvtKindUpdateRecord, + Collection: collection, + Rkey: rkey, + RecCid: &cc, + } + + if rm.hydrateRecords { + op.Record = rec + } + rm.events(ctx, &RepoEvent{ - User: user, - OldRoot: oldroot, - NewRoot: nroot, - Rev: nrev, - Since: &rev, - Ops: []RepoOp{{ - Kind: EvtKindUpdateRecord, - Collection: collection, - Rkey: rkey, - Record: rec, - RecCid: &cc, - }}, + User: user, + OldRoot: oldroot, + NewRoot: nroot, + Rev: nrev, + Since: &rev, + Ops: []RepoOp{op}, RepoSlice: rslice, }) } @@ -372,16 +379,21 @@ func (rm *RepoManager) InitNewActor(ctx context.Context, user models.Uid, handle } if rm.events != nil { + op := RepoOp{ + Kind: EvtKindCreateRecord, + Collection: "app.bsky.actor.profile", + Rkey: "self", + } + + if rm.hydrateRecords { + op.Record = profile + } + rm.events(ctx, &RepoEvent{ - User: user, - NewRoot: root, - Rev: nrev, - Ops: []RepoOp{{ - Kind: EvtKindCreateRecord, - Collection: "app.bsky.actor.profile", - Rkey: "self", - Record: profile, - }}, + User: user, + NewRoot: root, + Rev: nrev, + Ops: []RepoOp{op}, RepoSlice: rslice, }) } @@ -522,31 +534,40 @@ func (rm *RepoManager) HandleExternalUserEvent(ctx context.Context, pdsid uint, switch EventKind(op.Action) { case EvtKindCreateRecord: - recid, rec, err := r.GetRecord(ctx, op.Path) - if err != nil { - return fmt.Errorf("reading changed record from car slice: %w", err) - } - - evtops = append(evtops, RepoOp{ + rop := RepoOp{ Kind: EvtKindCreateRecord, Collection: parts[0], Rkey: parts[1], - Record: rec, - RecCid: &recid, - }) - case EvtKindUpdateRecord: - recid, rec, err := r.GetRecord(ctx, op.Path) - if err != nil { - return fmt.Errorf("reading changed record from car slice: %w", err) + RecCid: (*cid.Cid)(op.Cid), } - evtops = append(evtops, RepoOp{ + if rm.hydrateRecords { + _, rec, err := r.GetRecord(ctx, op.Path) + if err != nil { + return fmt.Errorf("reading changed record from car slice: %w", err) + } + rop.Record = rec + } + + evtops = append(evtops, rop) + case EvtKindUpdateRecord: + rop := RepoOp{ Kind: EvtKindUpdateRecord, Collection: parts[0], Rkey: parts[1], - Record: rec, - RecCid: &recid, - }) + RecCid: (*cid.Cid)(op.Cid), + } + + if rm.hydrateRecords { + _, rec, err := r.GetRecord(ctx, op.Path) + if err != nil { + return fmt.Errorf("reading changed record from car slice: %w", err) + } + + rop.Record = rec + } + + evtops = append(evtops, rop) case EvtKindDeleteRecord: evtops = append(evtops, RepoOp{ Kind: EvtKindDeleteRecord, @@ -624,13 +645,18 @@ func (rm *RepoManager) BatchWrite(ctx context.Context, user models.Uid, writes [ return err } - ops = append(ops, RepoOp{ + op := RepoOp{ Kind: EvtKindCreateRecord, Collection: c.Collection, Rkey: rkey, RecCid: &cc, - Record: c.Value.Val, - }) + } + + if rm.hydrateRecords { + op.Record = c.Value.Val + } + + ops = append(ops, op) case w.RepoApplyWrites_Update != nil: u := w.RepoApplyWrites_Update @@ -639,13 +665,18 @@ func (rm *RepoManager) BatchWrite(ctx context.Context, user models.Uid, writes [ return err } - ops = append(ops, RepoOp{ + op := RepoOp{ Kind: EvtKindUpdateRecord, Collection: u.Collection, Rkey: u.Rkey, RecCid: &cc, - Record: u.Value.Val, - }) + } + + if rm.hydrateRecords { + op.Record = u.Value.Val + } + + ops = append(ops, op) case w.RepoApplyWrites_Delete != nil: d := w.RepoApplyWrites_Delete @@ -740,7 +771,7 @@ func (rm *RepoManager) ImportNewRepo(ctx context.Context, user models.Uid, repoD var ops []RepoOp for _, op := range diffops { repoOpsImported.Inc() - out, err := processOp(ctx, bs, op) + out, err := processOp(ctx, bs, op, rm.hydrateRecords) if err != nil { log.Errorw("failed to process repo op", "err", err, "path", op.Rpath, "repo", repoDid) } @@ -776,7 +807,7 @@ func (rm *RepoManager) ImportNewRepo(ctx context.Context, user models.Uid, repoD return nil } -func processOp(ctx context.Context, bs blockstore.Blockstore, op *mst.DiffOp) (*RepoOp, error) { +func processOp(ctx context.Context, bs blockstore.Blockstore, op *mst.DiffOp, hydrateRecords bool) (*RepoOp, error) { parts := strings.SplitN(op.Rpath, "/", 2) if len(parts) != 2 { return nil, fmt.Errorf("repo mst had invalid rpath: %q", op.Rpath) @@ -784,10 +815,6 @@ func processOp(ctx context.Context, bs blockstore.Blockstore, op *mst.DiffOp) (* switch op.Op { case "add", "mut": - blk, err := bs.Get(ctx, op.NewCid) - if err != nil { - return nil, err - } kind := EvtKindCreateRecord if op.Op == "mut" { @@ -801,15 +828,22 @@ func processOp(ctx context.Context, bs blockstore.Blockstore, op *mst.DiffOp) (* RecCid: &op.NewCid, } - rec, err := lexutil.CborDecodeValue(blk.RawData()) - if err != nil { - if !errors.Is(err, lexutil.ErrUnrecognizedType) { + if hydrateRecords { + blk, err := bs.Get(ctx, op.NewCid) + if err != nil { return nil, err } - log.Warnf("failed processing repo diff: %s", err) - } else { - outop.Record = rec + rec, err := lexutil.CborDecodeValue(blk.RawData()) + if err != nil { + if !errors.Is(err, lexutil.ErrUnrecognizedType) { + return nil, err + } + + log.Warnf("failed processing repo diff: %s", err) + } else { + outop.Record = rec + } } return outop, nil diff --git a/testing/utils.go b/testing/utils.go index 10a8467b..a59a7c62 100644 --- a/testing/utils.go +++ b/testing/utils.go @@ -445,7 +445,7 @@ func SetupBGS(ctx context.Context, didr plc.PLCClient) (*TestBGS, error) { if err := ix.HandleRepoEvent(ctx, evt); err != nil { fmt.Println("test bgs failed to handle repo event", err) } - }) + }, true) // TODO: actually want this to be false, but some tests use this to confirm the BGS has seen certain records tr := &api.TestHandleResolver{} -- 2.51.2 From 1180330763529ed40709eee4fbbcbb4024a5c22c Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Thu, 12 Oct 2023 18:03:06 -0700 Subject: [PATCH 187/192] fixup --- cmd/supercollider/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/supercollider/main.go b/cmd/supercollider/main.go index 295ef0a4..9b1d2505 100644 --- a/cmd/supercollider/main.go +++ b/cmd/supercollider/main.go @@ -272,7 +272,7 @@ func Reload(cctx *cli.Context) error { TotalDesiredEvents: cctx.Int("total-events"), } - repoman.SetEventHandler(s.HandleRepoEvent) + repoman.SetEventHandler(s.HandleRepoEvent, false) // HTTP Server setup and Middleware Plumbing e := echo.New() -- 2.51.2 From de7c38a0383d6c7698a47ed92a75aae10a4d2a85 Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Fri, 13 Oct 2023 09:43:44 -0700 Subject: [PATCH 188/192] drop shard count threshold for compactions down to 20 --- bgs/compactor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bgs/compactor.go b/bgs/compactor.go index a3cd5c49..da6fc052 100644 --- a/bgs/compactor.go +++ b/bgs/compactor.go @@ -334,7 +334,7 @@ func (c *Compactor) EnqueueAllRepos(ctx context.Context, bgs *BGS, lim int, shar ) if shardCount == 0 { - shardCount = 50 + shardCount = 20 } span.SetAttributes(attribute.Int("clampedShardCount", shardCount)) -- 2.51.2 From 1ccae1beed26ac21208435cc4eb3456c58f9c8fe Mon Sep 17 00:00:00 2001 From: Emily Date: Sun, 15 Oct 2023 20:41:25 -0400 Subject: [PATCH 189/192] Add error handling for invite tree --- cmd/gosky/admin.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/gosky/admin.go b/cmd/gosky/admin.go index a954b20c..792d5dba 100644 --- a/cmd/gosky/admin.go +++ b/cmd/gosky/admin.go @@ -595,7 +595,8 @@ var listInviteTreeCmd = &cli.Command{ rep, err := atproto.AdminGetRepo(ctx, xrpcc, next) if err != nil { - return fmt.Errorf("getRepo %s: %w", did, err) + fmt.Printf("Failed to getRepo for DID %s: %s\n", next, err.Error()) + continue } fmt.Print(next) -- 2.51.2 From d34dfeaa3730d53692bc61c3e5157e911bcb5caa Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 16 Oct 2023 14:45:45 -0400 Subject: [PATCH 190/192] make the did cache not tiny --- cmd/bigsky/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 391d3b26..d0373083 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -256,7 +256,7 @@ func Bigsky(cctx *cli.Context) error { } mr.AddHandler("web", &webr) - cachedidr := plc.NewCachingDidResolver(mr, time.Minute*5, 1000) + cachedidr := plc.NewCachingDidResolver(mr, time.Minute*5, 500000) kmgr := indexer.NewKeyManager(cachedidr, nil) -- 2.51.2 From 9a60477489d748b82524640ace3f735fafbaf33d Mon Sep 17 00:00:00 2001 From: whyrusleeping Date: Mon, 16 Oct 2023 14:52:22 -0400 Subject: [PATCH 191/192] pedantry --- carstore/bs.go | 7 +++++++ cmd/bigsky/main.go | 2 +- repomgr/bench_test.go | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/carstore/bs.go b/carstore/bs.go index e7ba0b79..7a8bbedb 100644 --- a/carstore/bs.go +++ b/carstore/bs.go @@ -12,6 +12,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "github.com/bluesky-social/indigo/models" @@ -165,6 +166,9 @@ func (uv *userView) Has(ctx context.Context, k cid.Cid) (bool, error) { return count > 0, nil } +var CacheHits int64 +var CacheMiss int64 + func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, error) { if !k.Defined() { return nil, fmt.Errorf("attempted to 'get' undefined cid") @@ -172,9 +176,12 @@ func (uv *userView) Get(ctx context.Context, k cid.Cid) (blockformat.Block, erro if uv.cache != nil { blk, ok := uv.cache[k] if ok { + atomic.AddInt64(&CacheHits, 1) + return blk, nil } } + atomic.AddInt64(&CacheMiss, 1) // TODO: for now, im using a join to ensure we only query blocks from the // correct user. maybe it makes sense to put the user in the blockRef diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index d0373083..197b901c 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -256,7 +256,7 @@ func Bigsky(cctx *cli.Context) error { } mr.AddHandler("web", &webr) - cachedidr := plc.NewCachingDidResolver(mr, time.Minute*5, 500000) + cachedidr := plc.NewCachingDidResolver(mr, time.Minute*5, 500_000) kmgr := indexer.NewKeyManager(cachedidr, nil) diff --git a/repomgr/bench_test.go b/repomgr/bench_test.go index 3123f4ca..27181390 100644 --- a/repomgr/bench_test.go +++ b/repomgr/bench_test.go @@ -2,6 +2,7 @@ package repomgr import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -74,4 +75,6 @@ func BenchmarkRepoMgrCreates(b *testing.B) { b.Fatal(err) } } + + fmt.Println(carstore.CacheHits, carstore.CacheMiss) } -- 2.51.2 From 64ed9e5e8de7f456aa066818a2b912ddce9ef327 Mon Sep 17 00:00:00 2001 From: Jaz Volpert Date: Wed, 18 Oct 2023 19:36:53 +0000 Subject: [PATCH 192/192] Extend BGS identity cache duration to something reasonable and bust cache for a DID when getting an event from another PDS for a given repo --- bgs/bgs.go | 3 +++ cmd/bigsky/main.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/bgs/bgs.go b/bgs/bgs.go index efa10e3a..9c1eec8a 100644 --- a/bgs/bgs.go +++ b/bgs/bgs.go @@ -810,6 +810,9 @@ func (bgs *BGS) handleFedEvent(ctx context.Context, host *models.PDS, env *event if host.ID != u.PDS && u.PDS != 0 { log.Warnw("received event for repo from different pds than expected", "repo", evt.Repo, "expPds", u.PDS, "gotPds", host.Host) + // Flush any cached DID documents for this user + bgs.didr.FlushCacheFor(env.RepoHandle.Did) + subj, err := bgs.createExternalUser(ctx, evt.Repo) if err != nil { return err diff --git a/cmd/bigsky/main.go b/cmd/bigsky/main.go index 197b901c..6415e089 100644 --- a/cmd/bigsky/main.go +++ b/cmd/bigsky/main.go @@ -256,7 +256,7 @@ func Bigsky(cctx *cli.Context) error { } mr.AddHandler("web", &webr) - cachedidr := plc.NewCachingDidResolver(mr, time.Minute*5, 500_000) + cachedidr := plc.NewCachingDidResolver(mr, time.Hour*24, 500_000) kmgr := indexer.NewKeyManager(cachedidr, nil)