diff --git a/internal/db/db.go b/internal/db/db.go index a431332..14f69dd 100644 --- a/internal/db/db.go +++ b/internal/db/db.go @@ -2,7 +2,6 @@ package db import ( "context" - "sync" "gorm.io/gorm" "gorm.io/gorm/clause" @@ -10,31 +9,23 @@ import ( type DB struct { cli *gorm.DB - mu sync.Mutex } func NewDB(cli *gorm.DB) *DB { return &DB{ cli: cli, - mu: sync.Mutex{}, } } func (db *DB) Create(ctx context.Context, value any, clauses []clause.Expression) *gorm.DB { - db.mu.Lock() - defer db.mu.Unlock() return db.cli.WithContext(ctx).Clauses(clauses...).Create(value) } func (db *DB) Save(ctx context.Context, value any, clauses []clause.Expression) *gorm.DB { - db.mu.Lock() - defer db.mu.Unlock() return db.cli.WithContext(ctx).Clauses(clauses...).Save(value) } func (db *DB) Exec(ctx context.Context, sql string, clauses []clause.Expression, values ...any) *gorm.DB { - db.mu.Lock() - defer db.mu.Unlock() return db.cli.WithContext(ctx).Clauses(clauses...).Exec(sql, values...) } @@ -47,8 +38,6 @@ func (db *DB) AutoMigrate(models ...any) error { } func (db *DB) Delete(ctx context.Context, value any, clauses []clause.Expression) *gorm.DB { - db.mu.Lock() - defer db.mu.Unlock() return db.cli.WithContext(ctx).Clauses(clauses...).Delete(value) } @@ -56,17 +45,10 @@ func (db *DB) First(ctx context.Context, dest any, conds ...any) *gorm.DB { return db.cli.WithContext(ctx).First(dest, conds...) } -// TODO: this isn't actually good. we can commit even if the db is locked here. this is probably okay for the time being, but need to figure -// out a better solution. right now we only do this whenever we're importing a repo though so i'm mostly not worried, but it's still bad. -// e.g. when we do apply writes we should also be using a transcation but we don't right now -func (db *DB) BeginDangerously(ctx context.Context) *gorm.DB { +func (db *DB) Begin(ctx context.Context) *gorm.DB { return db.cli.WithContext(ctx).Begin() } -func (db *DB) Lock() { - db.mu.Lock() -} - -func (db *DB) Unlock() { - db.mu.Unlock() +func (db *DB) Client() *gorm.DB { + return db.cli } diff --git a/server/handle_import_repo.go b/server/handle_import_repo.go index 2ff4433..0e571a3 100644 --- a/server/handle_import_repo.go +++ b/server/handle_import_repo.go @@ -66,7 +66,7 @@ func (s *Server) handleRepoImportRepo(e echo.Context) error { return helpers.ServerError(e, nil) } - tx := s.db.BeginDangerously(ctx) + tx := s.db.Begin(ctx) clock := syntax.NewTIDClock(0) diff --git a/server/handle_server_delete_account.go b/server/handle_server_delete_account.go index 6ed80f3..066b6e4 100644 --- a/server/handle_server_delete_account.go +++ b/server/handle_server_delete_account.go @@ -69,7 +69,7 @@ func (s *Server) handleServerDeleteAccount(e echo.Context) error { }) } - tx := s.db.BeginDangerously(ctx) + tx := s.db.Begin(ctx) if tx.Error != nil { logger.Error("error starting transaction", "error", tx.Error) return helpers.ServerError(e, nil) diff --git a/server/server.go b/server/server.go index 2ad717b..09163c3 100644 --- a/server/server.go +++ b/server/server.go @@ -322,6 +322,9 @@ func New(args *Args) (*Server, error) { if err != nil { return nil, fmt.Errorf("failed to open sqlite database: %w", err) } + gdb.Exec("PRAGMA journal_mode=WAL") + gdb.Exec("PRAGMA synchronous=NORMAL") + logger.Info("connected to SQLite database", "path", args.DbName) } dbw := db.NewDB(gdb) @@ -625,68 +628,55 @@ func (s *Server) doBackup() { logger.Info("beginning backup to s3...") - var buf bytes.Buffer - if err := func() error { - logger.Info("reading database bytes...") - s.db.Lock() - defer s.db.Unlock() - - sf, err := os.Open(s.dbName) - if err != nil { - return fmt.Errorf("error opening database for backup: %w", err) - } - defer sf.Close() - - if _, err := io.Copy(&buf, sf); err != nil { - return fmt.Errorf("error reading bytes of backup db: %w", err) - } + tmpFile := fmt.Sprintf("/tmp/cocoon-backup-%s.db", time.Now().Format(time.RFC3339Nano)) + defer os.Remove(tmpFile) - return nil - }(); err != nil { - logger.Error("error backing up database", "error", err) + if err := s.db.Client().Exec(fmt.Sprintf("VACUUM INTO '%s'", tmpFile)).Error; err != nil { + logger.Error("error creating tmp backup file", "err", err) return } - if err := func() error { - logger.Info("sending to s3...") - - currTime := time.Now().Format("2006-01-02_15-04-05") - key := "cocoon-backup-" + currTime + ".db" + backupData, err := os.ReadFile(tmpFile) + if err != nil { + logger.Error("error reading tmp backup file", "err", err) + return + } - config := &aws.Config{ - Region: aws.String(s.s3Config.Region), - Credentials: credentials.NewStaticCredentials(s.s3Config.AccessKey, s.s3Config.SecretKey, ""), - } + logger.Info("sending to s3...") - if s.s3Config.Endpoint != "" { - config.Endpoint = aws.String(s.s3Config.Endpoint) - config.S3ForcePathStyle = aws.Bool(true) - } + currTime := time.Now().Format("2006-01-02_15-04-05") + key := "cocoon-backup-" + currTime + ".db" - sess, err := session.NewSession(config) - if err != nil { - return err - } + config := &aws.Config{ + Region: aws.String(s.s3Config.Region), + Credentials: credentials.NewStaticCredentials(s.s3Config.AccessKey, s.s3Config.SecretKey, ""), + } - svc := s3.New(sess) + if s.s3Config.Endpoint != "" { + config.Endpoint = aws.String(s.s3Config.Endpoint) + config.S3ForcePathStyle = aws.Bool(true) + } - if _, err := svc.PutObject(&s3.PutObjectInput{ - Bucket: aws.String(s.s3Config.Bucket), - Key: aws.String(key), - Body: bytes.NewReader(buf.Bytes()), - }); err != nil { - return fmt.Errorf("error uploading file to s3: %w", err) - } + sess, err := session.NewSession(config) + if err != nil { + logger.Error("error creating s3 session", "err", err) + return + } - logger.Info("finished uploading backup to s3", "key", key, "duration", time.Now().Sub(start).Seconds()) + svc := s3.New(sess) - return nil - }(); err != nil { - logger.Error("error uploading database backup", "error", err) + if _, err := svc.PutObject(&s3.PutObjectInput{ + Bucket: aws.String(s.s3Config.Bucket), + Key: aws.String(key), + Body: bytes.NewReader(backupData), + }); err != nil { + logger.Error("error uploading file to s3", "err", err) return } - os.WriteFile("last-backup.txt", []byte(time.Now().String()), 0644) + logger.Info("finished uploading backup to s3", "key", key, "duration", time.Since(start).Seconds()) + + os.WriteFile("last-backup.txt", []byte(time.Now().Format(time.RFC3339Nano)), 0644) } func (s *Server) backupRoutine() { @@ -721,10 +711,10 @@ func (s *Server) backupRoutine() { if err != nil { shouldBackupNow = true } else { - lastBackup, err := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", string(lastBackupStr)) + lastBackup, err := time.Parse(time.RFC3339Nano, string(lastBackupStr)) if err != nil { shouldBackupNow = true - } else if time.Now().Sub(lastBackup).Seconds() > 3600 { + } else if time.Since(lastBackup).Seconds() > 3600 { shouldBackupNow = true } } diff --git a/sqlite_blockstore/sqlite_blockstore.go b/sqlite_blockstore/sqlite_blockstore.go index 13a66cf..db1788f 100644 --- a/sqlite_blockstore/sqlite_blockstore.go +++ b/sqlite_blockstore/sqlite_blockstore.go @@ -94,7 +94,7 @@ func (bs *SqliteBlockstore) GetSize(context.Context, cid.Cid) (int, error) { } func (bs *SqliteBlockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { - tx := bs.db.BeginDangerously(ctx) + tx := bs.db.Begin(ctx) for _, block := range blocks { bs.inserts[block.Cid()] = block