From 74d723a7511c1536933e2d40d1de0dafbf47ac90 Mon Sep 17 00:00:00 2001 From: Will Andrews Date: Thu, 11 Dec 2025 16:00:03 +0000 Subject: [PATCH] update the env variables to be clearer and update the readme Signed-off-by: Will Andrews --- .env.example | 8 ++++---- main.go | 8 ++++---- readme.md | 22 +++++++++++++++++++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.env.example b/.env.example index aa948b5..7d66919 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,7 @@ -ENDPOINT="S3-endpoint" -ACCESS_ID="S3-ID" -SECRET_ACCESS_KEY="S3-secret" -BUCKET_NAME="my-super-duper-bucket" +S3_ENDPOINT="S3-endpoint" +S3_ACCESS_ID="S3-ID" +S3_SECRET_ACCESS_KEY="S3-secret" +S3_BUCKET_NAME="my-super-duper-bucket" DID="the-did-to-backup" PDS_HOST="https://your-pds.com" TANGLED_KNOT_DATABASE_DIRECTORY="/path/to/database/directory" diff --git a/main.go b/main.go index b6fb42d..36a4cc7 100644 --- a/main.go +++ b/main.go @@ -31,7 +31,7 @@ func main() { return } - bucketName := os.Getenv("BUCKET_NAME") + bucketName := os.Getenv("S3_BUCKET_NAME") err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{}) if err != nil { @@ -45,9 +45,9 @@ func main() { } func createMinioClient() (*minio.Client, error) { - endpoint := os.Getenv("ENDPOINT") - accessKeyID := os.Getenv("ACCESS_ID") - secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + endpoint := os.Getenv("S3_ENDPOINT") + accessKeyID := os.Getenv("S3_ACCESS_ID") + secretAccessKey := os.Getenv("S3_SECRET_ACCESS_KEY") useSSL := true return minio.New(endpoint, &minio.Options{ diff --git a/readme.md b/readme.md index f09d688..c32f830 100644 --- a/readme.md +++ b/readme.md @@ -2,8 +2,28 @@ This is a tool I'm activly developing to back up my ATProtocol type things to S3 storage. -At the moment it's a one shot style script that backs up the PDS repo and then the blobs but in the future I plan on being able to backup other things (next is my Tangled Knot data). +Currently there are 2 things that can be backed up: + +1: PDS data - A users repo and their blobs +2: Tangled Knot data - The repositories directory that contains all of the repo data and the directory that contains the SQLite database The PDS repo data is pulled straight from the xrpc endpoint at sent straight to S3. The blob data however is streamed into a zip file and sent to S3 so that not all the data is held in memory while the backup takes place (the minio library will still keep some in memory as a multipart request). It's very hacky right now and needs polishing to use with caution. Although let's face it, the worst it can do at the moment it backup some bad data which is better than no data 🤪 + + +### How to use + +Clone the repo and copy the `.env.example` file to be `.env`. Fill in the `.env` file with you S3 variables. + +For PDS data backup you need to ensure that `DID` and `PDS_HOST` are populated. + +For Knot data backup you need to ensure that `TANGLED_KNOT_DATABASE_DIRECTORY` and `TANGLED_KNOT_REPOSITORY_DIRECTORY` are populated. + +Run `go run .` + +### Todo + +- [ ] - Turn this into a long running app using a cron library perhaps +- [ ] - User query params properly when creating the URLs to fetch repo and blobs +- [ ] - Allow configuring the backup of knot repo data per users DID maybe? -- 2.51.2