From cee2c7fdcee5f321e89037f7bdbce42c8b82f0f8 Mon Sep 17 00:00:00 2001 From: Ewan Croft Date: Sun, 16 Nov 2025 16:29:23 +0000 Subject: [PATCH] feat: add AT Proto testing, force-sync mode, and improved sync logic --- README.md | 24 +++- SETUP.md | 175 +++++++++++++++++++++++++++ USAGE.md | 197 ++++++++++++++++++++++++++++++ package-lock.json | 4 +- package.json | 8 +- src/.env.example | 15 +++ src/check.ts | 266 +++++++++++++++++++++++++++++++++++++++++ src/index.ts | 175 ++++++++++++++++++++++----- src/test-atproto.ts | 71 +++++++++++ src/validate-config.ts | 109 +++++++++++++++++ tsconfig.json | 10 +- 11 files changed, 1013 insertions(+), 41 deletions(-) create mode 100644 SETUP.md create mode 100644 USAGE.md create mode 100644 src/.env.example create mode 100644 src/check.ts create mode 100644 src/test-atproto.ts create mode 100644 src/validate-config.ts diff --git a/README.md b/README.md index 90042da..3184a5c 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,13 @@ This tool is particularly useful for developers and organisations that want a de ### Configuration -Before running any scripts, you need to configure the project. See `src/.env` (or `src/config.env` if you prefer to keep a template) for the required environment variables: +Before running any scripts, you need to configure the project. Create a `src/.env` file based on `src/.env.example`: + +```bash +cp src/.env.example src/.env +``` + +Then edit `src/.env` with your actual values: * `BASE_DIR` – the local directory where GitHub repositories will be cloned. * `GITHUB_USER` – your GitHub username or organisation. @@ -45,9 +51,23 @@ Without proper SSH authentication, repository creation and pushing will fail. --- +### Testing AT Proto Connection + +**Before running the full sync**, test your AT Proto connection: + +```bash +npm run test-atproto +``` + +This will: +- Verify your Bluesky credentials +- Confirm your DID matches the configuration +- List any existing `sh.tangled.repo` records +- Validate the connection to the PDS + ### Running the Sync Script -Once configuration and SSH verification are complete, run: +Once configuration, SSH verification, and AT Proto testing are complete, run: ```bash npm run sync diff --git a/SETUP.md b/SETUP.md new file mode 100644 index 0000000..4448683 --- /dev/null +++ b/SETUP.md @@ -0,0 +1,175 @@ +# Tangled Sync - Setup & Troubleshooting Guide + +## Quick Setup Checklist + +### 1. Install Dependencies +```bash +npm install +``` + +### 2. Configure Environment Variables +```bash +# Copy the example env file +cp src/.env.example src/.env + +# Edit with your actual values +nano src/.env # or use your preferred editor +``` + +**Required values:** +- `BASE_DIR`: Where to clone repos (e.g., `/Users/you/tangled-repos`) +- `GITHUB_USER`: Your GitHub username +- `ATPROTO_DID`: Your AT Proto DID (get from Bluesky settings) +- `BLUESKY_PDS`: Usually `https://bsky.social` +- `BLUESKY_USERNAME`: Your Bluesky handle (e.g., `you.bsky.social`) +- `BLUESKY_PASSWORD`: Use an **app password**, not your main password! + +### 3. Get Your AT Proto DID + +Your DID can be found by: +1. Go to https://bsky.app +2. Click your profile +3. Settings → Advanced → Account +4. Look for "DID" (starts with `did:plc:`) + +Alternatively, visit: `https://bsky.social/xrpc/com.atproto.identity.resolveHandle?handle=YOUR_HANDLE.bsky.social` + +### 4. Create an App Password + +**IMPORTANT:** Do NOT use your main Bluesky password! + +1. Go to https://bsky.app/settings +2. Navigate to "App Passwords" +3. Click "Add App Password" +4. Give it a name (e.g., "Tangled Sync") +5. Copy the generated password to your `.env` file + +### 5. Test AT Proto Connection +```bash +npm run test-atproto +``` + +Expected output: +``` +✓ Login successful! + DID: did:plc:... + Handle: you.bsky.social + Email: your@email.com + +✓ Found X existing Tangled repo records +``` + +### 6. Verify SSH to Tangled +```bash +ssh git@tangled.sh +``` + +You should see a message confirming your SSH key is configured. + +### 7. Run the Sync +```bash +npm run sync +``` + +--- + +## Common Issues & Solutions + +### Issue: "Missing Bluesky credentials" +**Solution:** Check that `src/.env` exists and contains `BLUESKY_USERNAME` and `BLUESKY_PASSWORD` + +### Issue: "Login failed" or "Invalid credentials" +**Solution:** +- Ensure you're using an **app password**, not your main password +- Check your username includes the full handle (e.g., `you.bsky.social`) +- Verify credentials are correct + +### Issue: "DID mismatch" +**Solution:** +- Run `npm run test-atproto` to see your actual DID +- Update `ATPROTO_DID` in `src/.env` to match + +### Issue: "Could not push to Tangled" +**Solution:** +- Verify SSH key is added to Tangled: https://tangled.org/settings/keys +- Test SSH connection: `ssh git@tangled.sh` +- Ensure the repository exists on Tangled first + +### Issue: "Failed to create ATProto record" +**Solution:** +- Check that the schema matches (required fields: `name`, `knot`, `createdAt`) +- Verify your app password has write permissions +- Check PDS is reachable: `curl https://bsky.social` + +### Issue: Rate limiting from GitHub API +**Solution:** +- GitHub has a rate limit of 60 requests/hour for unauthenticated requests +- Consider adding GitHub authentication if syncing many repos +- Wait an hour and try again + +--- + +## Understanding the Workflow + +1. **Login to AT Proto**: Authenticates with Bluesky PDS using your credentials +2. **Fetch GitHub Repos**: Retrieves all public repos from your GitHub account +3. **Clone Locally**: Downloads repos to `BASE_DIR` if not already present +4. **Add Tangled Remote**: Adds `tangled` as a git remote +5. **Push to Tangled**: Pushes the `main` branch to Tangled +6. **Update README**: Adds a Tangled mirror link to the README +7. **Create AT Proto Record**: Publishes metadata to the AT Proto network + +Each repository gets a record in the `sh.tangled.repo` collection with: +- Repository name and description +- Source URL (GitHub) +- Creation timestamp +- Knot server reference +- Optional labels and topics + +--- + +## Verifying Success + +After running the sync, you can verify: + +1. **Local repos**: Check `BASE_DIR` for cloned repositories +2. **Tangled remotes**: Run `git remote -v` in any repo directory +3. **AT Proto records**: Run `npm run test-atproto` to list records +4. **Tangled website**: Visit `https://tangled.org/YOUR_DID/REPO_NAME` + +--- + +## Advanced Configuration + +### Using a Different PDS +If you're not using the default Bluesky PDS: +```bash +BLUESKY_PDS=https://your-pds.example.com +``` + +### Syncing Specific Repos Only +Modify the `getGitHubRepos()` function to filter repos: +```typescript +return json + .filter((r: any) => r.name.startsWith('my-prefix-')) + .map(...); +``` + +### Changing the Default Branch +If your repos use `master` instead of `main`, update: +```typescript +run(`git push tangled master`, repoDir); +``` + +--- + +## Support + +For issues with: +- **Tangled**: https://github.com/tangled-dev/tangled +- **AT Proto**: https://atproto.com/docs +- **This tool**: Open an issue in the repository + +--- + +**Happy syncing! 🚀** diff --git a/USAGE.md b/USAGE.md new file mode 100644 index 0000000..3540144 --- /dev/null +++ b/USAGE.md @@ -0,0 +1,197 @@ +# 🎉 Tangled Sync - Ready to Use! + +## Summary of Changes + +I've improved your Tangled Sync project to ensure proper AT Proto authentication and repository record creation. Here's what was updated: + +### ✅ What's Fixed + +1. **Enhanced AT Proto Login** + - Added better error handling and validation + - Shows DID and handle on successful login + - Clearer error messages when authentication fails + +2. **Corrected Repository Schema** + - Fixed record structure to match `sh.tangled.repo` lexicon + - Required fields (`name`, `knot`, `createdAt`) now ordered correctly + - Optional fields properly marked as optional + - Added better error handling for record creation + +3. **Improved Logging** + - More detailed startup information + - Better progress tracking during sync + - Shows AT Proto record URIs when created + - Success/failure messages are clearer + +### 📁 New Files Created + +1. **`src/.env.example`** - Template for your configuration +2. **`src/test-atproto.ts`** - Test AT Proto connection before syncing +3. **`src/validate-config.ts`** - Validate your environment setup +4. **`SETUP.md`** - Comprehensive setup and troubleshooting guide + +### 🚀 How to Use + +#### Step 1: Configure Environment +```bash +# Copy the example file +cp src/.env.example src/.env + +# Edit with your actual values +nano src/.env +``` + +You need: +- Your GitHub username +- Your AT Proto DID (from Bluesky settings) +- A Bluesky **app password** (not your main password!) +- Base directory for repos + +#### Step 2: Validate Configuration +```bash +npm run validate +``` + +This checks all your environment variables are set correctly. + +#### Step 3: Test AT Proto Connection +```bash +npm run test-atproto +``` + +This verifies: +- ✅ Your credentials work +- ✅ Your DID is correct +- ✅ You can access the PDS +- ✅ Shows any existing Tangled repo records + +#### Step 4: Run the Sync +```bash +npm run sync +``` + +This will: +1. Login to AT Proto ✅ +2. Fetch your GitHub repos +3. Clone them locally (if needed) +4. Add Tangled remotes +5. Push to Tangled +6. Update READMEs +7. Create AT Proto records for each repo ✅ + +### 🔍 What to Check + +After running the sync, verify: + +1. **AT Proto Records Created** + ```bash + npm run test-atproto + ``` + Should show your repos listed + +2. **Repos on Tangled** + Visit: `https://tangled.org/YOUR_DID/REPO_NAME` + +3. **Local Git Remotes** + ```bash + cd YOUR_BASE_DIR/some-repo + git remote -v + ``` + Should show both `origin` (GitHub) and `tangled` remotes + +### 📊 Record Schema + +Each repository creates a record with this structure: + +```typescript +{ + $type: "sh.tangled.repo", + name: "your-repo-name", // required + knot: "knot1.tangled.sh", // required + createdAt: "2024-01-01T00:00:00Z", // required + description: "Repo description", // optional + source: "https://github.com/...", // optional + labels: [], // optional +} +``` + +This matches the official `sh.tangled.repo` lexicon schema. + +### ⚠️ Important Notes + +1. **Use App Password**: Never use your main Bluesky password. Create an app password in Settings → App Passwords. + +2. **Check Your DID**: Run `npm run test-atproto` first to ensure your DID in `.env` matches your actual account. + +3. **SSH Key Required**: Make sure your SSH key is added to Tangled at https://tangled.org/settings/keys + +4. **Rate Limits**: GitHub API has rate limits (60 req/hour unauthenticated). If you have many repos, consider adding GitHub auth. + +### 🐛 Troubleshooting + +**"Missing Bluesky credentials"** +- Check `src/.env` exists and has `BLUESKY_USERNAME` and `BLUESKY_PASSWORD` + +**"Login failed"** +- Verify you're using an app password, not your main password +- Check username includes full handle (e.g., `you.bsky.social`) + +**"Could not push to Tangled"** +- Verify SSH key is configured: `ssh git@tangled.sh` +- Check repo exists on Tangled + +**"Failed to create ATProto record"** +- Run `npm run test-atproto` to check connection +- Verify your app password has write permissions + +See `SETUP.md` for more detailed troubleshooting. + +### 📚 Available Commands + +```bash +npm run check # Comprehensive health check (recommended first step!) +npm run validate # Check environment configuration only +npm run test-atproto # Test AT Proto connection only +npm run sync # Run sync (only new repos without AT Proto records) +npm run sync:force # Force sync all repos (including existing) +``` + +#### `npm run check` - Comprehensive Health Check + +This is the **most useful command** for troubleshooting! It runs all checks in one go: + +- ✅ Configuration validation +- ✅ AT Proto connection test +- ✅ SSH connection to Tangled +- ✅ GitHub API access +- ✅ Dependencies verification + +**When to use:** +- Before your first sync +- When troubleshooting issues +- After changing configuration +- To verify everything is working + +#### Individual Check Commands + +**Normal sync** (recommended): Only processes repos that don't have AT Proto records yet. This is efficient and safe for regular use. + +**Force sync**: Processes all repos regardless of whether they already have records. Use this if you need to: +- Re-push repos to Tangled +- Update READMEs for all repos +- Recover from a partial sync + +### ✨ Next Steps + +1. Copy and configure `src/.env` +2. Run `npm run validate` +3. Run `npm run test-atproto` +4. Run `npm run sync` + +That's it! Your GitHub repos will be synced to Tangled with proper AT Proto records. + +--- + +**Questions?** Check `SETUP.md` for detailed instructions and troubleshooting. + +**Happy syncing! 🚀** diff --git a/package-lock.json b/package-lock.json index 673097d..6ef60cb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -306,6 +306,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "dev": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -328,8 +329,7 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.14.0.tgz", "integrity": "sha512-QQiYxHuyZ9gQUIrmPo3IA+hUl4KYk8uSA7cHrcKd/l3p1OTpZcM0Tbp9x7FAtXdAYhlasd60ncPpgu6ihG6TOA==", "dev": true, - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/v8-compile-cache-lib": { "version": "3.0.1", diff --git a/package.json b/package.json index e3af4ac..9a628eb 100644 --- a/package.json +++ b/package.json @@ -2,15 +2,21 @@ "name": "tangled-sync", "version": "1.0.0", "description": "Sync GitHub repos to Tangled with ATProto records", + "type": "module", "main": "src/index.ts", "scripts": { - "sync": "ts-node src/index.ts" + "check": "ts-node src/check.ts", + "validate": "ts-node src/validate-config.ts", + "test-atproto": "ts-node src/test-atproto.ts", + "sync": "ts-node src/index.ts", + "sync:force": "ts-node src/index.ts --force" }, "dependencies": { "@atproto/api": "^0.17.2", "dotenv": "^16.0.0" }, "devDependencies": { + "@types/node": "^20.0.0", "ts-node": "^10.9.2", "typescript": "^5.9.3" }, diff --git a/src/.env.example b/src/.env.example new file mode 100644 index 0000000..00665ee --- /dev/null +++ b/src/.env.example @@ -0,0 +1,15 @@ +# Base directory where GitHub repos will be cloned +BASE_DIR=/path/to/your/repos + +# Your GitHub username +GITHUB_USER=your-github-username + +# Your ATProto DID (e.g., did:plc:abc123...) +ATPROTO_DID=did:plc:your-did-here + +# Bluesky PDS URL (usually https://bsky.social) +BLUESKY_PDS=https://bsky.social + +# Your Bluesky credentials +BLUESKY_USERNAME=your-handle.bsky.social +BLUESKY_PASSWORD=your-app-password diff --git a/src/check.ts b/src/check.ts new file mode 100644 index 0000000..4775f85 --- /dev/null +++ b/src/check.ts @@ -0,0 +1,266 @@ +import { AtpAgent } from "@atproto/api"; +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; +import { execSync } from "child_process"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +dotenv.config({ path: "./src/.env" }); + +async function runHealthCheck() { + +console.log("🔍 Running Tangled Sync Health Check...\n"); + +const checks: { category: string; name: string; status: boolean; message: string }[] = []; +let errors = 0; +let warnings = 0; + +// ===== CONFIGURATION CHECKS ===== +console.log("📋 Configuration Checks\n"); + +const envPath = path.join(__dirname, ".env"); +const envExists = fs.existsSync(envPath); +checks.push({ + category: "config", + name: ".env file", + status: envExists, + message: envExists ? "Found at src/.env" : "Missing! Copy src/.env.example to src/.env" +}); +if (!envExists) errors++; + +const requiredVars = [ + { name: "BASE_DIR", description: "Base directory for repos" }, + { name: "GITHUB_USER", description: "GitHub username" }, + { name: "ATPROTO_DID", description: "AT Proto DID" }, + { name: "BLUESKY_PDS", description: "Bluesky PDS URL" }, + { name: "BLUESKY_USERNAME", description: "Bluesky username" }, + { name: "BLUESKY_PASSWORD", description: "Bluesky app password" }, +]; + +requiredVars.forEach(({ name, description }) => { + const value = process.env[name]; + const exists = !!value && value.trim().length > 0; + checks.push({ + category: "config", + name: name, + status: exists, + message: exists ? `Set` : `Missing (${description})` + }); + if (!exists) errors++; +}); + +// Check BASE_DIR +const baseDir = process.env.BASE_DIR; +if (baseDir) { + const baseDirExists = fs.existsSync(baseDir); + checks.push({ + category: "config", + name: "BASE_DIR path", + status: baseDirExists, + message: baseDirExists ? `Exists: ${baseDir}` : `Missing (will be created): ${baseDir}` + }); + if (!baseDirExists) warnings++; +} + +// Check DID format +const did = process.env.ATPROTO_DID; +if (did) { + const validDid = did.startsWith("did:plc:") || did.startsWith("did:web:"); + checks.push({ + category: "config", + name: "DID format", + status: validDid, + message: validDid ? "Valid" : "Invalid! Should start with 'did:plc:' or 'did:web:'" + }); + if (!validDid) errors++; +} + +// Check PDS URL +const pds = process.env.BLUESKY_PDS; +if (pds) { + const validPds = pds.startsWith("http://") || pds.startsWith("https://"); + checks.push({ + category: "config", + name: "PDS URL", + status: validPds, + message: validPds ? pds : "Invalid! Should start with 'https://'" + }); + if (!validPds) errors++; +} + +// Print config results +checks.filter(c => c.category === "config").forEach((check) => { + const icon = check.status ? "✅" : "❌"; + console.log(`${icon} ${check.name}: ${check.message}`); +}); + +// ===== AT PROTO CONNECTION CHECK ===== +console.log("\n🔐 AT Proto Connection Check\n"); + +const canTestConnection = process.env.BLUESKY_USERNAME && + process.env.BLUESKY_PASSWORD && + process.env.BLUESKY_PDS && + process.env.ATPROTO_DID; + +if (canTestConnection) { + try { + const agent = new AtpAgent({ service: process.env.BLUESKY_PDS! }); + + const loginResponse = await agent.login({ + identifier: process.env.BLUESKY_USERNAME!, + password: process.env.BLUESKY_PASSWORD! + }); + + console.log(`✅ Login successful`); + console.log(` DID: ${loginResponse.data.did}`); + console.log(` Handle: ${loginResponse.data.handle}`); + + if (loginResponse.data.did !== process.env.ATPROTO_DID) { + console.log(`⚠️ DID mismatch!`); + console.log(` Expected: ${process.env.ATPROTO_DID}`); + console.log(` Got: ${loginResponse.data.did}`); + warnings++; + } + + // Test fetching records + const records = await agent.api.com.atproto.repo.listRecords({ + repo: loginResponse.data.did, + collection: "sh.tangled.repo", + limit: 5, + }); + + console.log(`✅ Can access AT Proto records`); + console.log(` Found ${records.data.records.length} sample records`); + + } catch (error: any) { + console.log(`❌ AT Proto connection failed`); + console.log(` Error: ${error.message}`); + errors++; + } +} else { + console.log("⏭️ Skipped (missing credentials)"); +} + +// ===== SSH CONNECTION CHECK ===== +console.log("\n🔑 SSH Connection Check\n"); + +try { + const sshTest = execSync("ssh -T git@tangled.sh 2>&1", { + encoding: "utf-8", + timeout: 5000 + }); + + if (sshTest.includes("successfully authenticated") || sshTest.includes("Hi")) { + console.log("✅ SSH connection to Tangled works"); + console.log(` ${sshTest.trim().split('\n')[0]}`); + } else { + console.log("⚠️ SSH connection uncertain"); + console.log(` Response: ${sshTest.trim()}`); + warnings++; + } +} catch (error: any) { + const output = error.stdout?.toString() || error.message; + + if (output.includes("successfully authenticated") || output.includes("Hi")) { + console.log("✅ SSH connection to Tangled works"); + } else { + console.log("❌ SSH connection to Tangled failed"); + console.log(" Make sure your SSH key is added at https://tangled.org/settings/keys"); + errors++; + } +} + +// ===== GITHUB API CHECK ===== +console.log("\n🐙 GitHub API Check\n"); + +if (process.env.GITHUB_USER) { + try { + const response = execSync(`curl -s "https://api.github.com/users/${process.env.GITHUB_USER}"`, { + encoding: "utf-8", + timeout: 5000 + }); + + const data = JSON.parse(response); + + if (data.login) { + console.log(`✅ GitHub user found: ${data.login}`); + console.log(` Public repos: ${data.public_repos || 0}`); + } else { + console.log(`❌ GitHub user not found: ${process.env.GITHUB_USER}`); + errors++; + } + } catch (error: any) { + console.log(`⚠️ Could not check GitHub API`); + console.log(` ${error.message}`); + warnings++; + } +} else { + console.log("⏭️ Skipped (no GITHUB_USER set)"); +} + +// ===== DEPENDENCIES CHECK ===== +console.log("\n📦 Dependencies Check\n"); + +let hasAtproto = false; +let hasDotenv = false; + +try { + await import("@atproto/api"); + hasAtproto = true; + console.log("✅ @atproto/api installed"); +} catch { + console.log("❌ @atproto/api not installed (run: npm install)"); + errors++; +} + +try { + await import("dotenv"); + hasDotenv = true; + console.log("✅ dotenv installed"); +} catch { + console.log("❌ dotenv not installed (run: npm install)"); + errors++; +} + +// ===== SUMMARY ===== +console.log("\n" + "=".repeat(50)); + +if (errors === 0 && warnings === 0) { + console.log("✅ All checks passed! Ready to sync."); + console.log("\nNext steps:"); + console.log(" npm run sync # Sync new repos only"); + console.log(" npm run sync:force # Force sync all repos"); +} else { + if (errors > 0) { + console.log(`❌ ${errors} error(s) found - please fix before syncing`); + } + if (warnings > 0) { + console.log(`⚠️ ${warnings} warning(s) - review before syncing`); + } + + console.log("\nSee SETUP.md for detailed troubleshooting"); + + if (errors > 0) { + process.exit(1); + } +} + +console.log("=".repeat(50)); + +// Additional recommendations +if (process.env.BLUESKY_PASSWORD && !process.env.BLUESKY_PASSWORD.includes("-")) { + console.log("\n💡 Tip: Your password might be a regular password."); + console.log(" Consider using an App Password from Bluesky settings for better security."); +} + +} + +// Run the health check +runHealthCheck().catch((error) => { + console.error("\n❌ Health check failed with error:"); + console.error(error); + process.exit(1); +}); diff --git a/src/index.ts b/src/index.ts index e701c2d..aa7f354 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,8 @@ import { execSync } from "child_process"; dotenv.config({ path: "./src/.env" }); +const FORCE_SYNC = process.argv.includes("--force"); + const BASE_DIR = process.env.BASE_DIR!; const GITHUB_USER = process.env.GITHUB_USER!; const ATPROTO_DID = process.env.ATPROTO_DID!; @@ -17,9 +19,19 @@ const agent = new AtpAgent({ service: BLUESKY_PDS }); async function login() { const username = process.env.BLUESKY_USERNAME; const password = process.env.BLUESKY_PASSWORD; - if (!username || !password) throw new Error("Missing Bluesky credentials"); - await agent.login({ identifier: username, password }); - console.log("[LOGIN] Logged in to Bluesky"); + if (!username || !password) { + throw new Error("Missing Bluesky credentials. Please set BLUESKY_USERNAME and BLUESKY_PASSWORD in src/.env"); + } + + try { + const response = await agent.login({ identifier: username, password }); + console.log(`[LOGIN] Successfully logged in to AT Proto as ${response.data.did}`); + console.log(`[LOGIN] Session handle: ${response.data.handle}`); + return response; + } catch (error: any) { + console.error("[ERROR] Failed to login to AT Proto:", error.message); + throw error; + } } async function getGitHubRepos(): Promise<{ clone_url: string; name: string; description?: string }[]> { @@ -90,16 +102,18 @@ function generateTid(): string { return toBase32Sortable(tidBigInt); } -// Tangled repo schema typing +// Tangled repo schema typing (matches sh.tangled.repo lexicon) interface TangledRepoRecord { $type: "sh.tangled.repo"; - knot: string; - name: string; - spindle: string; - description: string; - source: string; - labels: string[]; - createdAt: string; + name: string; // required + knot: string; // required + createdAt: string; // required (ISO 8601 datetime) + spindle?: string; // optional CI runner + description?: string; // optional, max 140 graphemes + website?: string; // optional URI + topics?: string[]; // optional array of topics + source?: string; // optional source URI + labels?: string[]; // optional array of at-uri labels } // Cache for existing repo records @@ -111,8 +125,10 @@ async function ensureTangledRecord( githubUser: string, repoName: string, description?: string -): Promise { - if (recordCache[repoName]) return recordCache[repoName]; +): Promise<{ tid: string; existed: boolean }> { + if (recordCache[repoName]) { + return { tid: recordCache[repoName], existed: true }; + } let cursor: string | undefined = undefined; let tid: string | null = null; @@ -131,7 +147,7 @@ async function ensureTangledRecord( tid = record.rkey; recordCache[repoName] = tid; console.log(`[FOUND] Existing record for ${repoName} (TID: ${tid})`); - break; + return { tid, existed: true }; } } @@ -142,27 +158,33 @@ async function ensureTangledRecord( tid = generateTid(); const record: TangledRepoRecord = { $type: "sh.tangled.repo", - knot: "knot1.tangled.sh", name: repoName, - spindle: "", + knot: "knot1.tangled.sh", + createdAt: new Date().toISOString(), description: description ?? repoName, source: `https://github.com/${githubUser}/${repoName}`, labels: [], - createdAt: new Date().toISOString(), }; - await agent.api.com.atproto.repo.putRecord({ - repo: atprotoDid, - collection: "sh.tangled.repo", - rkey: tid, - record, - }); + try { + const result = await agent.api.com.atproto.repo.putRecord({ + repo: atprotoDid, + collection: "sh.tangled.repo", + rkey: tid, + record, + }); + console.log(`[CREATED] ATProto record URI: ${result.data.uri}`); + } catch (error: any) { + console.error(`[ERROR] Failed to create ATProto record for ${repoName}:`, error.message); + throw error; + } recordCache[repoName] = tid; console.log(`[CREATED] Tangled record for ${repoName} (TID: ${tid})`); + return { tid, existed: false }; } - return tid; + return { tid, existed: false }; } function updateReadme(baseDir: string, repoName: string, atprotoDid: string) { @@ -187,23 +209,110 @@ Mirrored on Tangled: https://tangled.org/${atprotoDid}/${repoName} } async function main() { + console.log("[STARTUP] Starting Tangled Sync..."); + if (FORCE_SYNC) { + console.log("[MODE] Force sync enabled - will process all repos"); + } + console.log(`[CONFIG] Base directory: ${BASE_DIR}`); + console.log(`[CONFIG] GitHub user: ${GITHUB_USER}`); + console.log(`[CONFIG] ATProto DID: ${ATPROTO_DID}`); + console.log(`[CONFIG] PDS: ${BLUESKY_PDS}`); + + // Login to AT Proto await login(); + + // Ensure base directory exists ensureDir(BASE_DIR); + + // Fetch GitHub repositories + console.log(`[GITHUB] Fetching repositories for ${GITHUB_USER}...`); const repos = await getGitHubRepos(); + console.log(`[GITHUB] Found ${repos.length} repositories`); + + let reposToProcess = repos; + let skippedRepos: typeof repos = []; + + if (!FORCE_SYNC) { + // Fetch all existing Tangled records upfront + console.log(`[ATPROTO] Fetching existing Tangled records...`); + let cursor: string | undefined = undefined; + const existingRepos = new Set(); + + do { + const res: any = await agent.api.com.atproto.repo.listRecords({ + repo: ATPROTO_DID, + collection: "sh.tangled.repo", + limit: 100, + cursor, + }); + + for (const record of res.data.records) { + const value = record.value as TangledRepoRecord; + if (value.name) { + existingRepos.add(value.name); + recordCache[value.name] = record.rkey; + } + } + + cursor = res.data.cursor; + } while (cursor); + + console.log(`[ATPROTO] Found ${existingRepos.size} existing Tangled records`); + + // Separate repos into new and existing + reposToProcess = repos.filter(r => !existingRepos.has(r.name)); + skippedRepos = repos.filter(r => existingRepos.has(r.name)); + + console.log(`[INFO] ${reposToProcess.length} new repos to sync`); + console.log(`[INFO] ${skippedRepos.length} repos already synced (skipping)\n`); + + if (skippedRepos.length > 0) { + console.log("[SKIPPED] The following repos already have AT Proto records:"); + skippedRepos.forEach(r => console.log(` - ${r.name}`)); + console.log(""); + } + } else { + console.log("[INFO] Processing all ${repos.length} repos (force sync mode)\n"); + } + + let syncedCount = 0; + let errorCount = 0; - for (const { clone_url, name: repoName, description } of repos) { - console.log(`[PROGRESS] Processing ${repoName}`); + for (const { clone_url, name: repoName, description } of reposToProcess) { + console.log(`\n[PROGRESS] Processing ${repoName} (${syncedCount + 1}/${reposToProcess.length})`); const repoDir = path.join(BASE_DIR, repoName); - if (!fs.existsSync(repoDir)) { - run(`git clone ${clone_url} ${repoDir}`); - console.log(`[CLONE] ${repoName}`); - } + try { + if (!fs.existsSync(repoDir)) { + run(`git clone ${clone_url} ${repoDir}`); + console.log(`[CLONE] ${repoName}`); + } else { + console.log(`[EXISTS] ${repoName} already cloned`); + } - await ensureTangledRemoteAndPush(repoDir, repoName, clone_url); - updateReadme(BASE_DIR, repoName, ATPROTO_DID); - await ensureTangledRecord(agent, ATPROTO_DID, GITHUB_USER, repoName, description); + await ensureTangledRemoteAndPush(repoDir, repoName, clone_url); + updateReadme(BASE_DIR, repoName, ATPROTO_DID); + const result = await ensureTangledRecord(agent, ATPROTO_DID, GITHUB_USER, repoName, description); + + if (!result.existed) { + syncedCount++; + } + } catch (error: any) { + console.error(`[ERROR] Failed to sync ${repoName}: ${error.message}`); + errorCount++; + } + } + + console.log(`\n${'='.repeat(50)}`); + console.log(`[COMPLETE] Sync finished!`); + console.log(` ✅ New repos synced: ${syncedCount}`); + if (!FORCE_SYNC) { + console.log(` ⏭️ Repos skipped: ${skippedRepos.length}`); + } + if (errorCount > 0) { + console.log(` ❌ Errors: ${errorCount}`); } + console.log(`${'='.repeat(50)}`); } main().catch(console.error); diff --git a/src/test-atproto.ts b/src/test-atproto.ts new file mode 100644 index 0000000..6cc1b5d --- /dev/null +++ b/src/test-atproto.ts @@ -0,0 +1,71 @@ +import { AtpAgent } from "@atproto/api"; +import dotenv from "dotenv"; + +dotenv.config({ path: "./src/.env" }); + +async function testAtProtoConnection() { + console.log("Testing AT Proto Connection...\n"); + + const service = process.env.BLUESKY_PDS || "https://bsky.social"; + const username = process.env.BLUESKY_USERNAME; + const password = process.env.BLUESKY_PASSWORD; + const atprotoDid = process.env.ATPROTO_DID; + + console.log(`Service: ${service}`); + console.log(`Username: ${username}`); + console.log(`Expected DID: ${atprotoDid}\n`); + + if (!username || !password) { + console.error("ERROR: Missing BLUESKY_USERNAME or BLUESKY_PASSWORD"); + process.exit(1); + } + + const agent = new AtpAgent({ service }); + + try { + console.log("Attempting login..."); + const loginResponse = await agent.login({ + identifier: username, + password + }); + + console.log("✓ Login successful!"); + console.log(` DID: ${loginResponse.data.did}`); + console.log(` Handle: ${loginResponse.data.handle}`); + console.log(` Email: ${loginResponse.data.email || "N/A"}`); + + if (loginResponse.data.did !== atprotoDid) { + console.warn(`\n⚠ WARNING: Logged in DID (${loginResponse.data.did}) does not match ATPROTO_DID in .env (${atprotoDid})`); + console.warn(" Please update your ATPROTO_DID in src/.env"); + } + + // Test fetching existing records + console.log("\nFetching existing sh.tangled.repo records..."); + const records = await agent.api.com.atproto.repo.listRecords({ + repo: loginResponse.data.did, + collection: "sh.tangled.repo", + limit: 10, + }); + + console.log(`✓ Found ${records.data.records.length} existing Tangled repo records`); + + if (records.data.records.length > 0) { + console.log("\nSample records:"); + records.data.records.slice(0, 3).forEach((record: any) => { + console.log(` - ${record.value.name} (${record.uri})`); + }); + } + + console.log("\n✓ AT Proto connection test completed successfully!"); + + } catch (error: any) { + console.error("\n✗ AT Proto connection test failed!"); + console.error(`Error: ${error.message}`); + if (error.status) { + console.error(`HTTP Status: ${error.status}`); + } + process.exit(1); + } +} + +testAtProtoConnection(); diff --git a/src/validate-config.ts b/src/validate-config.ts new file mode 100644 index 0000000..d8a4600 --- /dev/null +++ b/src/validate-config.ts @@ -0,0 +1,109 @@ +import dotenv from "dotenv"; +import fs from "fs"; +import path from "path"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +dotenv.config({ path: "./src/.env" }); + +console.log("🔍 Validating Tangled Sync Configuration...\n"); + +const checks: { name: string; status: boolean; message: string }[] = []; + +// Check .env file exists +const envPath = path.join(__dirname, ".env"); +const envExists = fs.existsSync(envPath); +checks.push({ + name: ".env file", + status: envExists, + message: envExists ? "Found at src/.env" : "Missing! Copy src/.env.example to src/.env" +}); + +// Check required environment variables +const requiredVars = [ + { name: "BASE_DIR", description: "Base directory for repos" }, + { name: "GITHUB_USER", description: "GitHub username" }, + { name: "ATPROTO_DID", description: "AT Proto DID" }, + { name: "BLUESKY_PDS", description: "Bluesky PDS URL" }, + { name: "BLUESKY_USERNAME", description: "Bluesky username" }, + { name: "BLUESKY_PASSWORD", description: "Bluesky app password" }, +]; + +requiredVars.forEach(({ name, description }) => { + const value = process.env[name]; + const exists = !!value && value.trim().length > 0; + checks.push({ + name: `${name}`, + status: exists, + message: exists ? `✓ Set (${description})` : `✗ Missing (${description})` + }); +}); + +// Validate BASE_DIR +const baseDir = process.env.BASE_DIR; +if (baseDir) { + const baseDirExists = fs.existsSync(baseDir); + checks.push({ + name: "BASE_DIR exists", + status: baseDirExists, + message: baseDirExists ? `Directory exists: ${baseDir}` : `Directory missing: ${baseDir} (will be created)` + }); +} + +// Validate DID format +const did = process.env.ATPROTO_DID; +if (did) { + const validDid = did.startsWith("did:plc:") || did.startsWith("did:web:"); + checks.push({ + name: "DID format", + status: validDid, + message: validDid ? "Valid DID format" : "Invalid! Should start with 'did:plc:' or 'did:web:'" + }); +} + +// Validate PDS URL +const pds = process.env.BLUESKY_PDS; +if (pds) { + const validPds = pds.startsWith("http://") || pds.startsWith("https://"); + checks.push({ + name: "PDS URL format", + status: validPds, + message: validPds ? `Valid URL: ${pds}` : "Invalid! Should start with 'https://'" + }); +} + +// Print results +console.log("Configuration Check Results:\n"); +let allPassed = true; + +checks.forEach((check) => { + const icon = check.status ? "✅" : "❌"; + console.log(`${icon} ${check.name}: ${check.message}`); + if (!check.status) allPassed = false; +}); + +console.log("\n" + "=".repeat(50) + "\n"); + +if (allPassed) { + console.log("✅ All checks passed! You're ready to run:"); + console.log(" npm run test-atproto # Test AT Proto connection"); + console.log(" npm run sync # Run the full sync"); +} else { + console.log("❌ Some checks failed. Please fix the issues above."); + console.log(" See SETUP.md for detailed instructions."); + process.exit(1); +} + +// Additional recommendations +console.log("\n💡 Recommendations:"); + +if (process.env.BLUESKY_PASSWORD && !process.env.BLUESKY_PASSWORD.includes("-")) { + console.log(" ⚠️ Your password looks like it might be a regular password."); + console.log(" Consider using an App Password from Bluesky settings."); +} + +console.log(" 📚 Read SETUP.md for detailed setup instructions"); +console.log(" 🔐 Never commit your .env file to version control"); +console.log(" 🔑 Make sure your SSH key is added to Tangled"); diff --git a/tsconfig.json b/tsconfig.json index 2f3a4ff..3146e5c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,8 +1,8 @@ { "compilerOptions": { "target": "ES2022", - "module": "ESNext", - "moduleResolution": "bundler", + "module": "NodeNext", + "moduleResolution": "NodeNext", "rootDir": "./src", "outDir": "./dist", "strict": true, @@ -18,5 +18,9 @@ "allowJs": false }, "include": ["src/**/*.ts"], - "exclude": ["node_modules", "dist"] + "exclude": ["node_modules", "dist"], + "ts-node": { + "esm": true, + "experimentalSpecifierResolution": "node" + } } -- 2.51.2