From fbe3ca5e8b5375d9093addef6453a7417ec93b97 Mon Sep 17 00:00:00 2001 From: Daniel Roe Date: Mon, 4 May 2026 13:44:34 +0200 Subject: [PATCH] feat: create tangled repo on enrolment via repo.create with source url --- package.json | 3 + pnpm-lock.yaml | 260 +++++++++++++++++++++++++++++ server/api/atproto/callback.get.ts | 8 + server/utils/github-app.ts | 31 ++++ server/utils/job-handlers.ts | 99 ++++++++++- server/utils/tangled-repo.ts | 132 +++++++++++++++ test/unit/tangled-repo.spec.ts | 193 +++++++++++++++++++++ 7 files changed, 719 insertions(+), 7 deletions(-) create mode 100644 server/utils/github-app.ts create mode 100644 server/utils/tangled-repo.ts create mode 100644 test/unit/tangled-repo.spec.ts diff --git a/package.json b/package.json index 1370826..42400cf 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "test:browser:update": "docker run --rm --network host -v $(pwd):/work/ -v /tmp/playwright-node-modules:/work/node_modules -w /work/ -it mcr.microsoft.com/playwright:v1.59.1-noble bash -c 'corepack enable && pnpm i && pnpm playwright test test/browser --update-snapshots'" }, "dependencies": { + "@atcute/tid": "^1.1.2", "@atproto/api": "^0.19.11", "@atproto/jwk-jose": "^0.1.11", "@atproto/oauth-client-node": "^0.3.17", @@ -43,6 +44,8 @@ "@nuxt/image": "^2.0.0", "@nuxt/scripts": "^1.0.6", "@nuxtjs/html-validator": "^2.1.0", + "@octokit/app": "^16.1.2", + "@octokit/auth-app": "^8.2.0", "@octokit/webhooks-methods": "^6.0.0", "drizzle-orm": "^0.45.2", "nuxt": "^4.4.4", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2e59243..6da3eb8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,6 +12,9 @@ importers: .: dependencies: + '@atcute/tid': + specifier: ^1.1.2 + version: 1.1.2 '@atproto/api': specifier: ^0.19.11 version: 0.19.11 @@ -39,6 +42,12 @@ importers: '@nuxtjs/html-validator': specifier: ^2.1.0 version: 2.1.0(@voidzero-dev/vite-plus-test@0.1.20)(magicast@0.5.2) + '@octokit/app': + specifier: ^16.1.2 + version: 16.1.2 + '@octokit/auth-app': + specifier: ^8.2.0 + version: 8.2.0 '@octokit/webhooks-methods': specifier: ^6.0.0 version: 6.0.0 @@ -112,6 +121,12 @@ importers: packages: + '@atcute/tid@1.1.2': + resolution: {integrity: sha512-bmPuOX/TOfcm/vsK9vM98spjkcx2wgd9S2PeK5oLgEr8IbNRPq7iMCAPzOL1nu5XAW3LlkOYQEbYRcw5vcQ37w==} + + '@atcute/time-ms@1.3.2': + resolution: {integrity: sha512-F+qOyR9pO55g1d/QmN+Gr+fimoUQQLusdGSB6pjV0wW5KPILR4oQ4e2ZhWzqUbeHLAgWvgoTTMsMDdz62Xa2tg==} + '@atproto-labs/did-resolver@0.2.6': resolution: {integrity: sha512-2K1bC04nI2fmgNcvof+yA28IhGlpWn2JKYlPa7To9JTKI45FINCGkQSGiL2nyXlyzDJJ34fZ1aq6/IRFIOIiqg==} @@ -1427,6 +1442,81 @@ packages: '@nuxtjs/html-validator@2.1.0': resolution: {integrity: sha512-ldo8ioSsH3OEumtgwDMokTxlhjgO9FxjJWViAxisq5l/wjvaVX8SYTQ02wjtQcQQPSvS6BwgypAp400RlyFHng==} + '@octokit/app@16.1.2': + resolution: {integrity: sha512-8j7sEpUYVj18dxvh0KWj6W/l6uAiVRBl1JBDVRqH1VHKAO/G5eRVl4yEoYACjakWers1DjUkcCHyJNQK47JqyQ==} + engines: {node: '>= 20'} + + '@octokit/auth-app@8.2.0': + resolution: {integrity: sha512-vVjdtQQwomrZ4V46B9LaCsxsySxGoHsyw6IYBov/TqJVROrlYdyNgw5q6tQbB7KZt53v1l1W53RiqTvpzL907g==} + engines: {node: '>= 20'} + + '@octokit/auth-oauth-app@9.0.3': + resolution: {integrity: sha512-+yoFQquaF8OxJSxTb7rnytBIC2ZLbLqA/yb71I4ZXT9+Slw4TziV9j/kyGhUFRRTF2+7WlnIWsePZCWHs+OGjg==} + engines: {node: '>= 20'} + + '@octokit/auth-oauth-device@8.0.3': + resolution: {integrity: sha512-zh2W0mKKMh/VWZhSqlaCzY7qFyrgd9oTWmTmHaXnHNeQRCZr/CXy2jCgHo4e4dJVTiuxP5dLa0YM5p5QVhJHbw==} + engines: {node: '>= 20'} + + '@octokit/auth-oauth-user@6.0.2': + resolution: {integrity: sha512-qLoPPc6E6GJoz3XeDG/pnDhJpTkODTGG4kY0/Py154i/I003O9NazkrwJwRuzgCalhzyIeWQ+6MDvkUmKXjg/A==} + engines: {node: '>= 20'} + + '@octokit/auth-token@6.0.0': + resolution: {integrity: sha512-P4YJBPdPSpWTQ1NU4XYdvHvXJJDxM6YwpS0FZHRgP7YFkdVxsWcpWGy/NVqlAA7PcPCnMacXlRm1y2PFZRWL/w==} + engines: {node: '>= 20'} + + '@octokit/auth-unauthenticated@7.0.3': + resolution: {integrity: sha512-8Jb1mtUdmBHL7lGmop9mU9ArMRUTRhg8vp0T1VtZ4yd9vEm3zcLwmjQkhNEduKawOOORie61xhtYIhTDN+ZQ3g==} + engines: {node: '>= 20'} + + '@octokit/core@7.0.6': + resolution: {integrity: sha512-DhGl4xMVFGVIyMwswXeyzdL4uXD5OGILGX5N8Y+f6W7LhC1Ze2poSNrkF/fedpVDHEEZ+PHFW0vL14I+mm8K3Q==} + engines: {node: '>= 20'} + + '@octokit/endpoint@11.0.3': + resolution: {integrity: sha512-FWFlNxghg4HrXkD3ifYbS/IdL/mDHjh9QcsNyhQjN8dplUoZbejsdpmuqdA76nxj2xoWPs7p8uX2SNr9rYu0Ag==} + engines: {node: '>= 20'} + + '@octokit/graphql@9.0.3': + resolution: {integrity: sha512-grAEuupr/C1rALFnXTv6ZQhFuL1D8G5y8CN04RgrO4FIPMrtm+mcZzFG7dcBm+nq+1ppNixu+Jd78aeJOYxlGA==} + engines: {node: '>= 20'} + + '@octokit/oauth-app@8.0.3': + resolution: {integrity: sha512-jnAjvTsPepyUaMu9e69hYBuozEPgYqP4Z3UnpmvoIzHDpf8EXDGvTY1l1jK0RsZ194oRd+k6Hm13oRU8EoDFwg==} + engines: {node: '>= 20'} + + '@octokit/oauth-authorization-url@8.0.0': + resolution: {integrity: sha512-7QoLPRh/ssEA/HuHBHdVdSgF8xNLz/Bc5m9fZkArJE5bb6NmVkDm3anKxXPmN1zh6b5WKZPRr3697xKT/yM3qQ==} + engines: {node: '>= 20'} + + '@octokit/oauth-methods@6.0.2': + resolution: {integrity: sha512-HiNOO3MqLxlt5Da5bZbLV8Zarnphi4y9XehrbaFMkcoJ+FL7sMxH/UlUsCVxpddVu4qvNDrBdaTVE2o4ITK8ng==} + engines: {node: '>= 20'} + + '@octokit/openapi-types@27.0.0': + resolution: {integrity: sha512-whrdktVs1h6gtR+09+QsNk2+FO+49j6ga1c55YZudfEG+oKJVvJLQi3zkOm5JjiUXAagWK2tI2kTGKJ2Ys7MGA==} + + '@octokit/openapi-webhooks-types@12.1.0': + resolution: {integrity: sha512-WiuzhOsiOvb7W3Pvmhf8d2C6qaLHXrWiLBP4nJ/4kydu+wpagV5Fkz9RfQwV2afYzv3PB+3xYgp4mAdNGjDprA==} + + '@octokit/plugin-paginate-rest@14.0.0': + resolution: {integrity: sha512-fNVRE7ufJiAA3XUrha2omTA39M6IXIc6GIZLvlbsm8QOQCYvpq/LkMNGyFlB1d8hTDzsAXa3OKtybdMAYsV/fw==} + engines: {node: '>= 20'} + peerDependencies: + '@octokit/core': '>=6' + + '@octokit/request-error@7.1.0': + resolution: {integrity: sha512-KMQIfq5sOPpkQYajXHwnhjCC0slzCNScLHs9JafXc4RAJI+9f+jNDlBNaIMTvazOPLgb4BnlhGJOTbnN0wIjPw==} + engines: {node: '>= 20'} + + '@octokit/request@10.0.8': + resolution: {integrity: sha512-SJZNwY9pur9Agf7l87ywFi14W+Hd9Jg6Ifivsd33+/bGUQIjNujdFiXII2/qSlN2ybqUHfp5xpekMEjIBTjlSw==} + engines: {node: '>= 20'} + + '@octokit/types@16.0.0': + resolution: {integrity: sha512-sKq+9r1Mm4efXW1FCk7hFSeJo4QKreL/tTbR0rz/qx/r1Oa2VV83LTA/H/MuCOX7uCIJmQVRKBcbmWoySjAnSg==} + '@octokit/webhooks-methods@6.0.0': resolution: {integrity: sha512-MFlzzoDJVw/GcbfzVC1RLR36QqkTLUf79vLVO3D+xn7r0QgxnFoLZgtrzxiQErAjFUOdH6fas2KeQJ1yr/qaXQ==} engines: {node: '>= 20'} @@ -1434,6 +1524,10 @@ packages: '@octokit/webhooks-types@7.6.1': resolution: {integrity: sha512-S8u2cJzklBC0FgTwWVLaM8tMrDuDMVE4xiTK4EYXM9GntyvrdbSoxqDQa+Fh57CCNApyIpyeqPhhFEmHPfrXgw==} + '@octokit/webhooks@14.2.0': + resolution: {integrity: sha512-da6KbdNCV5sr1/txD896V+6W0iamFWrvVl8cHkBSPT+YlvmT3DwXa4jxZnQc+gnuTEqSWbBeoSZYTayXH9wXcw==} + engines: {node: '>= 20'} + '@one-ini/wasm@0.1.1': resolution: {integrity: sha512-XuySG1E38YScSJoMlqovLru4KTUNSjgVTIjyh7qMX6aNN5HY5Ct5LhRJdxO79JtTzKfzV/bnWpz+zquYrISsvw==} @@ -2647,6 +2741,9 @@ packages: '@tybys/wasm-util@0.10.2': resolution: {integrity: sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==} + '@types/aws-lambda@8.10.161': + resolution: {integrity: sha512-rUYdp+MQwSFocxIOcSsYSF3YYYC/uUpMbCY/mbO21vGqfrEYvNSoPyKYDj6RhXXpPfS0KstW9RwG3qXh9sL7FQ==} + '@types/chai@5.2.3': resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} @@ -3152,6 +3249,9 @@ packages: engines: {node: '>=6.0.0'} hasBin: true + before-after-hook@4.0.0: + resolution: {integrity: sha512-q6tR3RPqIB1pMiTRMFcZwuG5T8vwp+vUvEG0vuI6B+Rikh5BfPp2fQ82c925FOs+b0lcFQ8CFrL+KbilfZFhOQ==} + bindings@1.5.0: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} @@ -3752,6 +3852,9 @@ packages: resolution: {integrity: sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==} engines: {node: '>=18'} + fast-content-type-parse@3.0.0: + resolution: {integrity: sha512-ZvLdcY8P+N8mGQJahJV5G4U88CSvT1rP8ApL6uETe88MBXrBHAkZlSEySdUlyztF7ccb+Znos3TFqaepHxdhBg==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -4190,6 +4293,9 @@ packages: json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + json-with-bigint@3.5.8: + resolution: {integrity: sha512-eq/4KP6K34kwa7TcFdtvnftvHCD9KvHOGGICWwMFc4dOOKF5t4iYqnfLK8otCRCRv06FXOzGGyqE8h8ElMvvdw==} + json5@2.2.3: resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==} engines: {node: '>=6'} @@ -5314,6 +5420,10 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + toad-cache@3.7.0: + resolution: {integrity: sha512-/m8M+2BJUpoJdgAHoG+baCwBT+tf2VraSfkBgl0Y00qIWt41DJ8R5B8nsEw0I58YwF5IZH6z24/2TobDKnqSWw==} + engines: {node: '>=12'} + toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -5400,6 +5510,12 @@ packages: oxc-parser: optional: true + universal-github-app-jwt@2.2.2: + resolution: {integrity: sha512-dcmbeSrOdTnsjGjUfAlqNDJrhxXizjAz94ija9Qw8YkZ1uu0d+GoZzyH+Jb9tIIqvGsadUfwg+22k5aDqqwzbw==} + + universal-user-agent@7.0.3: + resolution: {integrity: sha512-TmnEAEAsBJVZM/AADELsK76llnwcf9vMKuPz8JflO1frO8Lchitr0fNaN9d+Ap0BjKtqWqd/J17qeDnXh8CL2A==} + unplugin-utils@0.3.1: resolution: {integrity: sha512-5lWVjgi6vuHhJ526bI4nlCOmkCIF3nnfXkCMDeMJrtdvxTs6ZFCM8oNufGTsDbKv/tJ/xj8RpvXjRuPBZJuJog==} engines: {node: '>=20.19.0'} @@ -5775,6 +5891,12 @@ packages: snapshots: + '@atcute/tid@1.1.2': + dependencies: + '@atcute/time-ms': 1.3.2 + + '@atcute/time-ms@1.3.2': {} + '@atproto-labs/did-resolver@0.2.6': dependencies: '@atproto-labs/fetch': 0.2.3 @@ -7215,10 +7337,134 @@ snapshots: - magicast - vitest + '@octokit/app@16.1.2': + dependencies: + '@octokit/auth-app': 8.2.0 + '@octokit/auth-unauthenticated': 7.0.3 + '@octokit/core': 7.0.6 + '@octokit/oauth-app': 8.0.3 + '@octokit/plugin-paginate-rest': 14.0.0(@octokit/core@7.0.6) + '@octokit/types': 16.0.0 + '@octokit/webhooks': 14.2.0 + + '@octokit/auth-app@8.2.0': + dependencies: + '@octokit/auth-oauth-app': 9.0.3 + '@octokit/auth-oauth-user': 6.0.2 + '@octokit/request': 10.0.8 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + toad-cache: 3.7.0 + universal-github-app-jwt: 2.2.2 + universal-user-agent: 7.0.3 + + '@octokit/auth-oauth-app@9.0.3': + dependencies: + '@octokit/auth-oauth-device': 8.0.3 + '@octokit/auth-oauth-user': 6.0.2 + '@octokit/request': 10.0.8 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/auth-oauth-device@8.0.3': + dependencies: + '@octokit/oauth-methods': 6.0.2 + '@octokit/request': 10.0.8 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/auth-oauth-user@6.0.2': + dependencies: + '@octokit/auth-oauth-device': 8.0.3 + '@octokit/oauth-methods': 6.0.2 + '@octokit/request': 10.0.8 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/auth-token@6.0.0': {} + + '@octokit/auth-unauthenticated@7.0.3': + dependencies: + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + + '@octokit/core@7.0.6': + dependencies: + '@octokit/auth-token': 6.0.0 + '@octokit/graphql': 9.0.3 + '@octokit/request': 10.0.8 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + before-after-hook: 4.0.0 + universal-user-agent: 7.0.3 + + '@octokit/endpoint@11.0.3': + dependencies: + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/graphql@9.0.3': + dependencies: + '@octokit/request': 10.0.8 + '@octokit/types': 16.0.0 + universal-user-agent: 7.0.3 + + '@octokit/oauth-app@8.0.3': + dependencies: + '@octokit/auth-oauth-app': 9.0.3 + '@octokit/auth-oauth-user': 6.0.2 + '@octokit/auth-unauthenticated': 7.0.3 + '@octokit/core': 7.0.6 + '@octokit/oauth-authorization-url': 8.0.0 + '@octokit/oauth-methods': 6.0.2 + '@types/aws-lambda': 8.10.161 + universal-user-agent: 7.0.3 + + '@octokit/oauth-authorization-url@8.0.0': {} + + '@octokit/oauth-methods@6.0.2': + dependencies: + '@octokit/oauth-authorization-url': 8.0.0 + '@octokit/request': 10.0.8 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + + '@octokit/openapi-types@27.0.0': {} + + '@octokit/openapi-webhooks-types@12.1.0': {} + + '@octokit/plugin-paginate-rest@14.0.0(@octokit/core@7.0.6)': + dependencies: + '@octokit/core': 7.0.6 + '@octokit/types': 16.0.0 + + '@octokit/request-error@7.1.0': + dependencies: + '@octokit/types': 16.0.0 + + '@octokit/request@10.0.8': + dependencies: + '@octokit/endpoint': 11.0.3 + '@octokit/request-error': 7.1.0 + '@octokit/types': 16.0.0 + fast-content-type-parse: 3.0.0 + json-with-bigint: 3.5.8 + universal-user-agent: 7.0.3 + + '@octokit/types@16.0.0': + dependencies: + '@octokit/openapi-types': 27.0.0 + '@octokit/webhooks-methods@6.0.0': {} '@octokit/webhooks-types@7.6.1': {} + '@octokit/webhooks@14.2.0': + dependencies: + '@octokit/openapi-webhooks-types': 12.1.0 + '@octokit/request-error': 7.1.0 + '@octokit/webhooks-methods': 6.0.0 + '@one-ini/wasm@0.1.1': {} '@oxc-minify/binding-android-arm-eabi@0.128.0': @@ -7917,6 +8163,8 @@ snapshots: tslib: 2.8.1 optional: true + '@types/aws-lambda@8.10.161': {} + '@types/chai@5.2.3': dependencies: '@types/deep-eql': 4.0.2 @@ -8434,6 +8682,8 @@ snapshots: baseline-browser-mapping@2.10.27: {} + before-after-hook@4.0.0: {} + bindings@1.5.0: dependencies: file-uri-to-path: 1.0.0 @@ -9026,6 +9276,8 @@ snapshots: fake-indexeddb@6.2.5: {} + fast-content-type-parse@3.0.0: {} + fast-deep-equal@3.1.3: {} fast-fifo@1.3.2: {} @@ -9493,6 +9745,8 @@ snapshots: json-stable-stringify-without-jsonify@1.0.1: {} + json-with-bigint@3.5.8: {} + json5@2.2.3: {} keyv@4.5.4: @@ -10971,6 +11225,8 @@ snapshots: dependencies: is-number: 7.0.0 + toad-cache@3.7.0: {} + toidentifier@1.0.1: {} totalist@3.0.1: {} @@ -11058,6 +11314,10 @@ snapshots: optionalDependencies: oxc-parser: 0.128.0 + universal-github-app-jwt@2.2.2: {} + + universal-user-agent@7.0.3: {} + unplugin-utils@0.3.1: dependencies: pathe: 2.0.3 diff --git a/server/api/atproto/callback.get.ts b/server/api/atproto/callback.get.ts index e529456..3ae04c7 100644 --- a/server/api/atproto/callback.get.ts +++ b/server/api/atproto/callback.get.ts @@ -1,4 +1,5 @@ import { userIdentity } from '~~/server/db/schema' +import { enqueue } from '~~/server/utils/queue' import { generateAndPublishKey } from '~~/server/utils/tangled-pubkey' export default defineEventHandler(async event => { @@ -33,5 +34,12 @@ export default defineEventHandler(async event => { installationId, }) + // Backfill: enqueue a single job that walks the installation's repo list + // and fans out per-repo enrolment. Doing this in the worker (rather than + // inline here) keeps the OAuth callback fast regardless of repo count, and + // gives us proper retry semantics if pagination doesn't finish in one + // worker tick. + await enqueue('tangled.backfill-installation', { installationId, page: 1 }) + await sendRedirect(event, '/dashboard', 302) }) diff --git a/server/utils/github-app.ts b/server/utils/github-app.ts new file mode 100644 index 0000000..28acd21 --- /dev/null +++ b/server/utils/github-app.ts @@ -0,0 +1,31 @@ +import { App } from '@octokit/app' + +let cachedApp: App | undefined + +function useApp(): App { + if (cachedApp) return cachedApp + const appId = process.env.NUXT_GITHUB_APP_ID + const privateKey = process.env.NUXT_GITHUB_APP_PRIVATE_KEY + if (!appId || !privateKey) { + throw new Error('NUXT_GITHUB_APP_ID and NUXT_GITHUB_APP_PRIVATE_KEY must be set') + } + cachedApp = new App({ + appId, + // Vercel env vars escape newlines; restore them so PEM parsing works. + privateKey: privateKey.replaceAll('\\n', '\n'), + }) + return cachedApp +} + +export type InstallationOctokit = Awaited> + +/** Get an Octokit pre-authed for a specific GitHub App installation. */ +export async function installationOctokit(installationId: number): Promise { + const app = useApp() + return app.getInstallationOctokit(installationId) +} + +/** Test hook. */ +export function clearGitHubAppCache() { + cachedApp = undefined +} diff --git a/server/utils/job-handlers.ts b/server/utils/job-handlers.ts index 6f2affa..bab37ac 100644 --- a/server/utils/job-handlers.ts +++ b/server/utils/job-handlers.ts @@ -1,14 +1,22 @@ -import type { JobEnvelope } from './queue' +import { sql } from 'drizzle-orm' +import { userIdentity } from '../db/schema' import { useOAuthClient } from './atproto-oauth' +import { useDb } from './db' +import { installationOctokit } from './github-app' +import type { JobEnvelope } from './queue' +import { enqueue } from './queue' import { generateAndPublishKey } from './tangled-pubkey' +import { enrollRepo } from './tangled-repo' /** - * Map of job kind → handler. Handlers are filled in by later commits: - * - 'github.push' → commit 12 (sync push events) - * - 'github.create' / 'github.delete' → commit 13 (branch/tag ref ops) - * - 'github.repository' → commit 14/15 (description, lifecycle) - * - 'tangled.create-repo' → commit 10 (initial enrolment) - * - 'atproto.publish-pubkey' → this commit (key rotation) + * Map of job kind → handler. Each commit fills in its slice: + * - 'github.push' → commit 12 (sync push events) + * - 'github.create' / 'github.delete' → commit 13 (branch/tag ref ops) + * - 'github.repository' → commit 14/15 (description, lifecycle) + * - 'github.installation_repositories' → this commit (fan-out enrolment) + * - 'tangled.backfill-installation' → this commit (paginate + fan-out) + * - 'tangled.create-repo' → this commit (per-repo enrolment) + * - 'atproto.publish-pubkey' → commit 9 * * Unknown kinds throw so they surface as job failures rather than silent * acknowledgement. @@ -19,15 +27,35 @@ const KNOWN_KINDS = new Set([ 'github.delete', 'github.repository', 'github.installation_repositories', + 'tangled.backfill-installation', 'tangled.create-repo', 'atproto.publish-pubkey', ]) +const BACKFILL_PAGE_SIZE = 100 + interface PublishPubkeyPayload { did: string installationId: number } +interface CreateRepoPayload { + installationId: number + githubRepoId: number +} + +interface InstallationRepositoriesPayload { + installationId: number + action: 'added' | 'removed' + addedRepoIds: number[] + removedRepoIds: number[] +} + +interface BackfillInstallationPayload { + installationId: number + page: number +} + export async function dispatch(envelope: JobEnvelope): Promise { if (!KNOWN_KINDS.has(envelope.kind)) { throw new Error(`unknown job kind: ${envelope.kind}`) @@ -41,5 +69,62 @@ export async function dispatch(envelope: JobEnvelope): Promise { return } + if (envelope.kind === 'tangled.create-repo') { + const { installationId, githubRepoId } = envelope.payload as CreateRepoPayload + + // Find the user identity bound to this install. If OAuth hasn't completed + // yet, drop this job silently \u2014 OAuth callback re-enqueues for all + // accessible repos at completion time, so we'll get a fresh trigger. + const db = useDb() + const identity = await db.select({ did: userIdentity.did }) + .from(userIdentity) + .where(sql`${userIdentity.installationId} = ${installationId}`) + if (identity.length === 0) return + + const client = await useOAuthClient() + const session = await client.restore(identity[0]!.did) + await enrollRepo({ oauthSession: session, installationId, githubRepoId }) + return + } + + if (envelope.kind === 'tangled.backfill-installation') { + const { installationId, page } = envelope.payload as BackfillInstallationPayload + const octokit = await installationOctokit(installationId) + const { data } = await octokit.request('GET /installation/repositories', { + per_page: BACKFILL_PAGE_SIZE, + page, + }) + + // Fan out one tangled.create-repo job per repo on this page. + for (const repo of data.repositories) { + // eslint-disable-next-line no-await-in-loop -- enqueue is sequential by design + await enqueue('tangled.create-repo', { installationId, githubRepoId: repo.id }) + } + + // If there are more pages, re-queue ourselves for the next one. This + // keeps each tick small and bounded; an install with thousands of repos + // walks through over many minutes rather than blocking one worker. + const seenSoFar = (page - 1) * BACKFILL_PAGE_SIZE + data.repositories.length + if (seenSoFar < data.total_count && data.repositories.length > 0) { + await enqueue('tangled.backfill-installation', { installationId, page: page + 1 }) + } + return + } + + if (envelope.kind === 'github.installation_repositories') { + const { installationId, action, addedRepoIds } = envelope.payload as InstallationRepositoriesPayload + if (action !== 'added') return + + // Fan out one tangled.create-repo job per added repo. The fan-out keeps + // each unit small enough to fit comfortably in the per-job lease, lets + // failures retry independently, and runs the OAuth precondition check + // per repo (an install can outlive a tangled identity disconnection). + for (const id of addedRepoIds) { + // eslint-disable-next-line no-await-in-loop -- fan-out enqueue is sequential by design + await enqueue('tangled.create-repo', { installationId, githubRepoId: id }) + } + return + } + // Other kinds: still no-op until handlers land in their commits. } diff --git a/server/utils/tangled-repo.ts b/server/utils/tangled-repo.ts new file mode 100644 index 0000000..6ad969e --- /dev/null +++ b/server/utils/tangled-repo.ts @@ -0,0 +1,132 @@ +import { Agent } from '@atproto/api' +import type { OAuthSession } from '@atproto/oauth-client-node' +import { now as tidNow } from '@atcute/tid' +import { sql } from 'drizzle-orm' +import { repoMapping } from '../db/schema' +import { useDb } from './db' +import { installationOctokit } from './github-app' + +const REPO_LEXICON = 'sh.tangled.repo' +const REPO_CREATE_NSID = 'sh.tangled.repo.create' + +/** + * Default knot for users with no `sh.tangled.knot` records. PLAN.md "Open + * questions" #1: confirm with the tangled team that this is the right + * appview-hosted default. + */ +const DEFAULT_KNOT = 'knot1.tangled.sh' + +export interface EnrolResult { + status: 'enrolled' | 'already' | 'skipped' + reason?: 'private' | 'fork' | 'no-identity' +} + +/** + * Enroll a single GitHub repo on tangled. + * + * Flow: + * 1. Skip if a `repo_mapping` row already exists. + * 2. Fetch GitHub repo metadata via the install token. Skip private/fork. + * 3. Pick a knot (user default → `DEFAULT_KNOT`). + * 4. Get a service-auth JWT for `(aud=did:web:, lxm=sh.tangled.repo.create)`. + * 5. POST to `https:///xrpc/sh.tangled.repo.create` with + * `{ rkey, name, source, defaultBranch }`. The knot clones the repo from + * `source` and mints a `repoDid`. + * 6. Write a `sh.tangled.repo` record on the user's PDS. + * 7. Insert the `repo_mapping` row. + */ +export async function enrollRepo(opts: { + oauthSession: OAuthSession + installationId: number + githubRepoId: number +}): Promise { + const db = useDb() + + const existing = await db.select({ id: repoMapping.id }) + .from(repoMapping) + .where(sql`${repoMapping.installationId} = ${opts.installationId} AND ${repoMapping.githubRepoId} = ${opts.githubRepoId}`) + if (existing.length > 0) { + return { status: 'already' } + } + + // 1. GitHub repo metadata. + const octokit = await installationOctokit(opts.installationId) + const { data: repo } = await octokit.request('GET /repositories/{repository_id}', { + repository_id: opts.githubRepoId, + }) + + if (repo.private) return { status: 'skipped', reason: 'private' } + if (repo.fork) return { status: 'skipped', reason: 'fork' } + + const [owner, name] = repo.full_name.split('/') + if (!owner || !name) { + throw new Error(`unexpected github full_name shape: ${repo.full_name}`) + } + + // 2. Pick a knot. Users *can* configure additional knots; v1 always uses + // the default. Wiring user choice through is dashboard work. + const knot = DEFAULT_KNOT + + // 3. Service-auth JWT for the knot procedure. + const agent = new Agent(opts.oauthSession) + const aud = `did:web:${knot}` + const exp = Math.floor(Date.now() / 1000) + 60 + const { data: { token } } = await agent.com.atproto.server.getServiceAuth({ + aud, + lxm: REPO_CREATE_NSID, + exp, + }) + + // 4. Knot procedure call. Tangled mints a repoDid here and starts cloning + // from `source`. + const rkey = tidNow() + const sourceUrl = `https://github.com/${owner}/${name}` + const knotResponse = await fetch(`https://${knot}/xrpc/${REPO_CREATE_NSID}`, { + method: 'POST', + headers: { + 'authorization': `Bearer ${token}`, + 'content-type': 'application/json', + }, + body: JSON.stringify({ + rkey, + name, + source: sourceUrl, + defaultBranch: repo.default_branch, + }), + }) + if (!knotResponse.ok) { + const body = await knotResponse.text() + throw new Error(`knot ${knot} returned ${knotResponse.status}: ${body}`) + } + const { repoDid } = await knotResponse.json() as { repoDid?: string } + if (!repoDid) { + throw new Error(`knot ${knot} returned no repoDid`) + } + + // 5. PDS record so the appview firehose discovers the repo. + await agent.com.atproto.repo.putRecord({ + repo: opts.oauthSession.did, + collection: REPO_LEXICON, + rkey, + record: { + $type: REPO_LEXICON, + name, + knot, + repoDid, + createdAt: new Date().toISOString(), + }, + }) + + // 6. Persist mapping. + await db.insert(repoMapping).values({ + installationId: opts.installationId, + githubRepoId: opts.githubRepoId, + githubFullName: repo.full_name, + tangledRepoDid: repoDid, + tangledFullName: `${opts.oauthSession.did}/${name}`, + knot, + status: 'active', + }) + + return { status: 'enrolled' } +} diff --git a/test/unit/tangled-repo.spec.ts b/test/unit/tangled-repo.spec.ts new file mode 100644 index 0000000..8735697 --- /dev/null +++ b/test/unit/tangled-repo.spec.ts @@ -0,0 +1,193 @@ +import crypto from 'node:crypto' +import { sql } from 'drizzle-orm' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { installation, repoMapping } from '../../server/db/schema' +import { clearDb, setDb, useDb } from '../../server/utils/db' +import { clearEncryptionKeyCache } from '../../server/utils/encryption' +import { enrollRepo } from '../../server/utils/tangled-repo' +import { createTestDb } from '../utils/db' + +const ORIGINAL_ENC_KEY = process.env.NUXT_ENCRYPTION_KEY + +interface GithubRepoLike { + id: number + full_name: string + private: boolean + fork: boolean + default_branch: string +} + +const githubGet = vi.fn<(input: { repository_id: number }) => Promise<{ data: GithubRepoLike }>>() +const getServiceAuthMock = vi.fn<(input: { aud: string, lxm: string, exp: number }) => Promise<{ data: { token: string } }>>() +const putRecordMock = vi.fn<(input: { repo: string, collection: string, rkey: string, record: Record }) => Promise>() + +vi.mock('@atproto/api', () => ({ + Agent: class { + com = { + atproto: { + server: { getServiceAuth: getServiceAuthMock }, + repo: { putRecord: putRecordMock }, + }, + } + }, +})) + +vi.mock('../../server/utils/github-app', () => ({ + installationOctokit: async () => ({ + request: githubGet, + }), + clearGitHubAppCache: () => {}, +})) + +const fakeFetch = vi.fn<(url: string, init: RequestInit) => Promise>() +const ORIGINAL_FETCH = globalThis.fetch + +describe('enrollRepo', () => { + beforeEach(async () => { + process.env.NUXT_ENCRYPTION_KEY = crypto.randomBytes(32).toString('base64') + clearEncryptionKeyCache() + + setDb(await createTestDb()) + await useDb().insert(installation).values({ + id: 1, accountLogin: 'alice', accountId: 100, accountType: 'User', + }) + + githubGet.mockReset() + getServiceAuthMock.mockReset() + putRecordMock.mockReset() + fakeFetch.mockReset() + globalThis.fetch = fakeFetch as unknown as typeof fetch + + getServiceAuthMock.mockResolvedValue({ data: { token: 'service-auth-jwt' } }) + putRecordMock.mockResolvedValue({ data: { uri: 'at://did:plc:abc/sh.tangled.repo/whatever', cid: 'bafy' } }) + }) + + afterEach(() => { + globalThis.fetch = ORIGINAL_FETCH + if (ORIGINAL_ENC_KEY === undefined) delete process.env.NUXT_ENCRYPTION_KEY + else process.env.NUXT_ENCRYPTION_KEY = ORIGINAL_ENC_KEY + clearEncryptionKeyCache() + clearDb() + }) + + function fakeOauthSession(did: string) { + return { did } as never + } + + function ghRepo(over: Partial = {}): GithubRepoLike { + return { + id: 9001, + full_name: 'alice/my-project', + private: false, + fork: false, + default_branch: 'main', + ...over, + } + } + + it('enrolls a public, non-fork repo end to end', async () => { + githubGet.mockResolvedValue({ data: ghRepo() }) + fakeFetch.mockResolvedValue(new Response( + JSON.stringify({ repoDid: 'did:plc:repo-xyz' }), + { status: 200 }, + )) + + const result = await enrollRepo({ + oauthSession: fakeOauthSession('did:plc:abc'), + installationId: 1, + githubRepoId: 9001, + }) + expect(result.status).toBe('enrolled') + + // Service auth requested with the right shape. + expect(getServiceAuthMock).toHaveBeenCalledTimes(1) + const sa = getServiceAuthMock.mock.calls[0]?.[0] + expect(sa?.aud).toBe('did:web:knot1.tangled.sh') + expect(sa?.lxm).toBe('sh.tangled.repo.create') + + // Knot procedure invoked with source URL and rkey. + expect(fakeFetch).toHaveBeenCalledTimes(1) + const fetchCall = fakeFetch.mock.calls[0] + const url = fetchCall?.[0] + const init = fetchCall?.[1] + expect(url).toBe('https://knot1.tangled.sh/xrpc/sh.tangled.repo.create') + expect((init!.headers as Record).authorization).toBe('Bearer service-auth-jwt') + const body = JSON.parse(init!.body as string) as Record + expect(body.name).toBe('my-project') + expect(body.source).toBe('https://github.com/alice/my-project') + expect(body.defaultBranch).toBe('main') + expect(typeof body.rkey).toBe('string') + + // PDS record written with the same rkey. + expect(putRecordMock).toHaveBeenCalledTimes(1) + const put = putRecordMock.mock.calls[0]?.[0] + expect(put?.rkey).toBe(body.rkey) + expect(put?.record.repoDid).toBe('did:plc:repo-xyz') + expect(put?.record.knot).toBe('knot1.tangled.sh') + + // Mapping persisted. + const rows = await useDb().select().from(repoMapping) + .where(sql`${repoMapping.installationId} = 1`) + expect(rows).toHaveLength(1) + expect(rows[0]!.tangledRepoDid).toBe('did:plc:repo-xyz') + expect(rows[0]!.knot).toBe('knot1.tangled.sh') + expect(rows[0]!.status).toBe('active') + }) + + it('skips private repos', async () => { + githubGet.mockResolvedValue({ data: ghRepo({ private: true }) }) + + const result = await enrollRepo({ + oauthSession: fakeOauthSession('did:plc:abc'), + installationId: 1, + githubRepoId: 9001, + }) + expect(result).toEqual({ status: 'skipped', reason: 'private' }) + expect(fakeFetch).not.toHaveBeenCalled() + expect(putRecordMock).not.toHaveBeenCalled() + expect(await useDb().select().from(repoMapping)).toHaveLength(0) + }) + + it('skips forks', async () => { + githubGet.mockResolvedValue({ data: ghRepo({ fork: true }) }) + + const result = await enrollRepo({ + oauthSession: fakeOauthSession('did:plc:abc'), + installationId: 1, + githubRepoId: 9001, + }) + expect(result).toEqual({ status: 'skipped', reason: 'fork' }) + expect(fakeFetch).not.toHaveBeenCalled() + }) + + it('no-ops if a mapping already exists', async () => { + await useDb().insert(repoMapping).values({ + installationId: 1, + githubRepoId: 9001, + githubFullName: 'alice/my-project', + status: 'active', + }) + + const result = await enrollRepo({ + oauthSession: fakeOauthSession('did:plc:abc'), + installationId: 1, + githubRepoId: 9001, + }) + expect(result).toEqual({ status: 'already' }) + expect(githubGet).not.toHaveBeenCalled() + }) + + it('throws and writes nothing if the knot rejects the procedure', async () => { + githubGet.mockResolvedValue({ data: ghRepo() }) + fakeFetch.mockResolvedValue(new Response('nope', { status: 500 })) + + await expect(enrollRepo({ + oauthSession: fakeOauthSession('did:plc:abc'), + installationId: 1, + githubRepoId: 9001, + })).rejects.toThrow(/knot1\.tangled\.sh returned 500/) + + expect(putRecordMock).not.toHaveBeenCalled() + expect(await useDb().select().from(repoMapping)).toHaveLength(0) + }) +}) -- 2.51.2