diff --git a/.env.example b/.env.example index 0a17398..b5a1ebb 100644 --- a/.env.example +++ b/.env.example @@ -4,7 +4,7 @@ S3_SECRET_ACCESS_KEY="S3-secret" S3_BUCKET_NAME="my-super-duper-bucket" DID="the-did-to-backup" PDS_HOST="https://your-pds.com" -TANGLED_KNOT_DATABASE_DIRECTORY="/path/to/database/directory" -TANGLED_KNOT_REPOSITORY_DIRECTORY="/path/to/repository/directory" +TANGLED_KNOT_DATABASE_DIRECTORY="./tangled/path/to/database/directory" # ./tangled is from the docker-compose volume +TANGLED_KNOT_REPOSITORY_DIRECTORY="./tangled/path/to/repository/directory" # ./tangled is from the docker-compose volume BUGSNAG_API_KEY="enter-api-key-to-enable" BLOB_DIR="./blobs" diff --git a/docker-compose.yaml b/docker-compose.yaml index 1176269..46b5947 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -3,7 +3,8 @@ services: container_name: back-at-it image: willdot/back-at-it:latest environment: - ENV_LOCATION: "/app/data/ back-at-it.env" + ENV_LOCATION: "/app/data/back-at-it.env" volumes: - ./data:/app/data + - /home/will/apps/tangled:/app/tangled:ro restart: always diff --git a/go.mod b/go.mod index b1b315d..abdbdf7 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,15 @@ module tangled.sh/willdot.net/backatit -go 1.25.0 +go 1.26.0 require ( + github.com/bugsnag/bugsnag-go/v2 v2.6.2 github.com/joho/godotenv v1.5.1 github.com/minio/minio-go/v7 v7.0.95 + github.com/robfig/cron v1.2.0 ) require ( - github.com/bugsnag/bugsnag-go/v2 v2.6.2 // indirect github.com/bugsnag/panicwrap v1.3.4 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ini/ini v1.67.0 // indirect @@ -21,7 +22,6 @@ require ( github.com/minio/md5-simd v1.1.2 // indirect github.com/philhofer/fwd v1.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/robfig/cron v1.2.0 // indirect github.com/rs/xid v1.6.0 // indirect github.com/stretchr/testify v1.10.0 // indirect github.com/tinylib/msgp v1.3.0 // indirect diff --git a/go.sum b/go.sum index e1efb3a..f815c00 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= github.com/bugsnag/bugsnag-go/v2 v2.6.2 h1:gGjr8txMtPYWKovEBC+4o6tthYveuE7fjzu6XYVIApg= github.com/bugsnag/bugsnag-go/v2 v2.6.2/go.mod h1:S9njhE7l6XCiKycOZ2zp0x1zoEE5nL3HjROCSsKc/3c= diff --git a/main.go b/main.go index a0e619f..af3feb7 100644 --- a/main.go +++ b/main.go @@ -5,6 +5,7 @@ import ( "log/slog" "net/http" "os" + "os/signal" "time" "github.com/bugsnag/bugsnag-go/v2" @@ -24,9 +25,15 @@ type service struct { } func main() { - ctx := context.Background() + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) + defer stop() - err := godotenv.Load(".env") + envLocation := os.Getenv("ENV_LOCATION") + if envLocation == "" { + envLocation = ".env" + } + + err := godotenv.Load(envLocation) if err != nil { if !os.IsNotExist(err) { slog.Error("load env", "error", err) @@ -83,6 +90,9 @@ func main() { }) c.Start() + defer c.Stop() + + <-ctx.Done() } func createMinioClient() (*minio.Client, error) { diff --git a/pds.go b/pds.go index cfd13e6..625a6e8 100644 --- a/pds.go +++ b/pds.go @@ -9,6 +9,7 @@ import ( "log/slog" "net/http" "os" + "path" "path/filepath" "time" @@ -54,7 +55,50 @@ func (s *service) backupRepo(ctx context.Context) error { defer resp.Body.Close() - _, err = s.minioClient.PutObject(ctx, s.bucketName, "pds-repo", resp.Body, -1, minio.PutObjectOptions{}) + b, err := io.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("reading repo response body: %w", err) + } + + filename := path.Join(s.blobDir, fmt.Sprintf("%s-%d-repo.car", s.did, time.Now().UnixMilli())) + f, err := os.Create(filename) + if err != nil { + return fmt.Errorf("creating temp file: %w", err) + } + defer func() { + f.Close() + + err = os.Remove(filename) + if err != nil { + slog.Error("failed to delete pds repo file after uploading", "error", err, "filename", f.Name()) + } + }() + + zipWriter := zip.NewWriter(f) + zipFile, err := zipWriter.Create("repo.car") + if err != nil { + return fmt.Errorf("create zip file: %w", err) + } + + _, err = zipFile.Write(b) + if err != nil { + return fmt.Errorf("write repo to file: %w", err) + } + + zipWriter.Close() + + // reset the reader back to the start so that the minio upload can read the data that's been written. + _, err = f.Seek(0, 0) + if err != nil { + return fmt.Errorf("setting seek on written file: %w", err) + } + + fi, err := f.Stat() + if err != nil { + return fmt.Errorf("stat file: %w", err) + } + + _, err = s.minioClient.PutObject(ctx, s.bucketName, "pds-repo.zip", f, fi.Size(), minio.PutObjectOptions{}) if err != nil { return fmt.Errorf("stream repo to bucket: %w", err) } diff --git a/readme.md b/readme.md index 032ab11..7eff1f8 100644 --- a/readme.md +++ b/readme.md @@ -18,9 +18,11 @@ Clone the repo and copy the `.env.example` file to be `.env`. Fill in the `.env` For PDS data backup you need to ensure that `DID` and `PDS_HOST` are populated. (You can run this tool on any machine to back PDS data up) -For Knot data backup you need to ensure that `TANGLED_KNOT_DATABASE_DIRECTORY` and `TANGLED_KNOT_REPOSITORY_DIRECTORY` are populated. (You need to run this tool on your Knot server to back up Knot data) +For Knot data backup you need to ensure that `TANGLED_KNOT_DATABASE_DIRECTORY` and `TANGLED_KNOT_REPOSITORY_DIRECTORY` are populated. (You need to run this tool on your Knot server to back up Knot data). -Run `go run .` +If using Docker, in the `docker-compose.yaml` file there is a mount that needs to be set which should point to the directory on your host machine that contains your Tangled Knot application running (it has read only permissions, don't worry). Then inside the `.env` file you need to set the `TANGLED_KNOT_DATABASE_DIRECTORY` and `TANGLED_KNOT_REPOSITORY_DIRECTORY` envs to be the directories inside that Knot application volume. + +Run `go run .` or use Docker. ### Todo diff --git a/tangled_knot.go b/tangled_knot.go index 7750355..a4e0836 100644 --- a/tangled_knot.go +++ b/tangled_knot.go @@ -4,10 +4,13 @@ import ( "archive/tar" "compress/gzip" "context" + "fmt" "io" "log/slog" "os" + "path" "path/filepath" + "time" "github.com/bugsnag/bugsnag-go/v2" "github.com/minio/minio-go/v7" @@ -26,12 +29,37 @@ func (s *service) backupKnotDB(ctx context.Context) { slog.Info("TANGLED_KNOT_DATABASE_DIRECTORY env not set - skipping knot DB backup") } - pipeReader, pipeWriter := io.Pipe() - defer pipeReader.Close() + filename := path.Join(s.blobDir, fmt.Sprintf("%d-knot.zip", time.Now().UnixMilli())) + f, err := os.Create(filename) + if err != nil { + slog.Error("creating temp file", "error", err) + return + } + defer func() { + f.Close() - go compress(dir, pipeWriter) + err = os.Remove(filename) + if err != nil { + slog.Error("failed to delete knot db zip file after uploading", "error", err, "filename", f.Name()) + } + }() - _, err := s.minioClient.PutObject(ctx, s.bucketName, "knot-db.zip", pipeReader, -1, minio.PutObjectOptions{}) + compress(dir, f) + + // reset the reader back to the start so that the minio upload can read the data that's been written. + _, err = f.Seek(0, 0) + if err != nil { + slog.Error("setting seek on written file", "error", err) + return + } + + fi, err := f.Stat() + if err != nil { + slog.Error("failed to stat knot db zip file", "error", err) + return + } + + _, err = s.minioClient.PutObject(ctx, s.bucketName, "knot-db.zip", f, fi.Size(), minio.PutObjectOptions{}) if err != nil { slog.Error("stream knot DB to bucket", "error", err) bugsnag.Notify(err) @@ -44,26 +72,47 @@ func (s *service) backupKnotRepos(ctx context.Context) { slog.Info("TANGLED_KNOT_REPOSITORY_DIRECTORY env not set - skipping knot repo backup") } - pipeReader, pipeWriter := io.Pipe() - defer pipeReader.Close() + filename := path.Join(s.blobDir, fmt.Sprintf("%d-knot-repos.zip", time.Now().UnixMilli())) + f, err := os.Create(filename) + if err != nil { + slog.Error("creating temp file", "error", err) + return + } + defer func() { + f.Close() - go compress(dir, pipeWriter) + err = os.Remove(filename) + if err != nil { + slog.Error("failed to delete knot repos zip file after uploading", "error", err, "filename", f.Name()) + } + }() + + compress(dir, f) + + // reset the reader back to the start so that the minio upload can read the data that's been written. + _, err = f.Seek(0, 0) + if err != nil { + slog.Error("setting seek on written file", "error", err) + return + } - _, err := s.minioClient.PutObject(ctx, s.bucketName, "knot-repos.zip", pipeReader, -1, minio.PutObjectOptions{}) + fi, err := f.Stat() if err != nil { - slog.Error("stream knot repos to bucket", "error", err) + slog.Error("failed to stat knot db zip file", "error", err) + return + } + + _, err = s.minioClient.PutObject(ctx, s.bucketName, "knot-repos.zip", f, fi.Size(), minio.PutObjectOptions{}) + if err != nil { + slog.Error("write knot repos to bucket", "error", err) bugsnag.Notify(err) } } -func compress(src string, writer io.WriteCloser) error { +func compress(src string, writer io.Writer) error { zipWriter := gzip.NewWriter(writer) tarWriter := tar.NewWriter(zipWriter) - defer writer.Close() - defer zipWriter.Close() - defer tarWriter.Close() - filepath.Walk(src, func(file string, fi os.FileInfo, err error) error { header, err := tar.FileInfoHeader(fi, file) if err != nil { @@ -87,6 +136,7 @@ func compress(src string, writer io.WriteCloser) error { return err } } + return nil }) @@ -98,6 +148,6 @@ func compress(src string, writer io.WriteCloser) error { if err := zipWriter.Close(); err != nil { return err } - // + return nil }