diff --git a/.gitignore b/.gitignore
index 36eee0a4d..168fae6fe 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,6 +33,8 @@ build/
.wrangler/
localinfra/certs/*
localinfra/vendor/
+__pycache__/
+*.pyc
id_rsa
id_ecdsa
id_dsa
diff --git a/docker-compose.yml b/docker-compose.yml
index 1f92b769d..e60401844 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -225,7 +225,7 @@ services:
KNOT_LEGACY_ADMIN_SECRET: super-strong-devonly-key
KNOT_PLC_DIRECTORY: https://plc.tngl.boltless.dev
KNOT_APPVIEW_ENDPOINT: https://tngl.boltless.dev
- KNOT_EXTRA_CA_FILE: /etc/ssl/certs/tngl.crt
+ KNOT_EXTRA_CA_FILE: /etc/ssl/certs/tngl.crt # cluster root
KNOT_XRPC_TRUSTED_PROXY_HEADER: x-forwarded-for
KNOT_XRPC_TRUSTED_PROXIES: 11.0.0.0/24
KNOT_LISTEN_REQUEST_TIMEOUT_MS: "600000"
@@ -562,8 +562,8 @@ services:
DELIBERI_PLC_URL: https://plc.tngl.boltless.dev
DELIBERI_JETSTREAM_ENDPOINT: wss://hydrant.tngl.boltless.dev/subscribe
DELIBERI_BOBBIN_API_URL: http://bobbin:8090
- # browser-reachable frontend; the web service replaced the appview frontend
- DELIBERI_BASE_URL: http://127.0.0.1:5174
+ # verification links in deliberi mail point here; must be browser-reachable
+ DELIBERI_BASE_URL: https://web.tngl.boltless.dev
DELIBERI_PDS_HOST: https://pds.tngl.boltless.dev
# PDS admin password for minting invite codes + creating accounts
DELIBERI_PDS_ADMIN_SECRET: ${TANGLED_PDS_ADMIN_SECRET}
@@ -714,6 +714,7 @@ services:
context: .
dockerfile: localinfra/web.Dockerfile
restart: unless-stopped
+ env_file: localinfra/certs/github.env
environment:
# cloudflare adapter → vite dev emulates the worker platform (env bindings)
# from web/.wrangler/state, so local KV seeding works like the deployed worker
@@ -736,6 +737,10 @@ services:
AVATAR_SHARED_SECRET: localinfra-avatar-secret
VITE_HANDLE_RESOLVER_URL: https://pds.tngl.boltless.dev
VITE_PLC_DIRECTORY_URL: https://plc.tngl.boltless.dev
+ MIGRATOR_URL: https://migrator.tngl.boltless.dev
+ GITHUB_PUBLIC_ORIGIN: https://github.tngl.boltless.dev
+ GITHUB_API_ORIGIN: https://github.tngl.boltless.dev
+ GITHUB_AVATARS_ORIGIN: https://github.tngl.boltless.dev
# public turnstile site key rendered by the /signup page widget
TURNSTILE_SITE_KEY: ${TANGLED_CLOUDFLARE_TURNSTILE_SITE_KEY}
# host-side port; keeps the oauth loopback redirect consistent
@@ -769,6 +774,56 @@ services:
condition: service_started
networks: [tngl, upstream-cache]
+ github-stub:
+ build:
+ context: .
+ dockerfile: localinfra/github-stub.Dockerfile
+ restart: unless-stopped
+ environment:
+ PORT: "8000"
+ REPOS_DIR: /data/repos
+ GITHUB_PUBLIC_ORIGIN: https://github.tngl.boltless.dev
+ GITHUB_API_ORIGIN: https://github.tngl.boltless.dev
+ GITHUB_AVATARS_ORIGIN: https://github.tngl.boltless.dev
+ GITHUB_APP_SETUP_URL: ${GITHUB_APP_SETUP_URL:-https://web.tngl.boltless.dev/_internal/github/install/callback}
+ volumes:
+ - github-stub-data:/data
+ healthcheck:
+ test: ["CMD", "python3", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:8000/user', timeout=2).status == 200 else 1)"]
+ interval: 5s
+ timeout: 3s
+ retries: 12
+ start_period: 5s
+ networks:
+ tngl:
+ aliases:
+ # deliberi validates committer email domains by DNS
+ - octostub.dev
+
+ migrator:
+ build:
+ context: .
+ dockerfile: localinfra/migrator.Dockerfile
+ restart: unless-stopped
+ env_file: ./localinfra/certs/migrator.env
+ environment:
+ MIGRATOR_HOSTNAME: migrator.tngl.boltless.dev
+ MIGRATOR_APP_URL: https://web.tngl.boltless.dev
+ MIGRATOR_PLC_URL: https://plc.tngl.boltless.dev
+ MIGRATOR_GITHUB_HOST: github.tngl.boltless.dev
+ MIGRATOR_DB_PATH: /var/lib/migrator/migrator.db
+ MIGRATOR_WORK_DIR: /var/lib/migrator/scratch
+ volumes:
+ - migrator-data:/var/lib/migrator
+ - ./localinfra/certs/root.crt:/usr/local/share/ca-certificates/caddy.crt:ro
+ healthcheck:
+ test: ["CMD", "wget", "-qO-", "http://localhost:6767/.well-known/did.json"]
+ interval: 5s
+ timeout: 2s
+ retries: 12
+ start_period: 5s
+ networks: [tngl]
+
camo:
dns: [11.0.0.254]
depends_on: [dns]
@@ -843,6 +898,8 @@ services:
- david.pds.tngl.boltless.dev
- hydrant.tngl.boltless.dev
- knot2.tngl.boltless.dev
+ # exercises the remote fetch path instead of same-authority
+ - knot2-alt.tngl.boltless.dev
- spindle.tngl.boltless.dev
- tngl.boltless.dev
- mirror.tngl.boltless.dev
@@ -857,6 +914,8 @@ services:
- delegates.tngl.boltless.dev
- tranquil.tngl.boltless.dev
- troy.tranquil.tngl.boltless.dev
+ - github.tngl.boltless.dev
+ - api.github.tngl.boltless.dev
prometheus:
image: prom/prometheus:v2.54.1
@@ -916,6 +975,8 @@ services:
networks: [tngl]
volumes:
+ migrator-data:
+ github-stub-data:
caddy-data:
postgres-data:
pds-data:
diff --git a/localinfra/Caddyfile b/localinfra/Caddyfile
index f9f7dac35..6981ce94d 100644
--- a/localinfra/Caddyfile
+++ b/localinfra/Caddyfile
@@ -62,7 +62,9 @@ hydrant.tngl.boltless.dev {
# doesn't import cors: a browser rejects a response with two Access-Control-Allow-Origin
# values, and web/ posts sh.tangled.git.keepCommit
# straight from the browser to the knot serving the repo.
-knot2.tngl.boltless.dev {
+# knot2-alt is the same knot under a second authority, so a local import exercises the
+# remote fetch path instead of the same-authority one
+knot2.tngl.boltless.dev, knot2-alt.tngl.boltless.dev {
tls internal
reverse_proxy knot2:5555
}
@@ -144,6 +146,17 @@ pocket.tngl.boltless.dev {
reverse_proxy pocket:3000
}
+# migrator
+migrator.tngl.boltless.dev {
+ tls internal
+ reverse_proxy migrator:6767
+}
+
+# github stub
+github.tngl.boltless.dev, api.github.tngl.boltless.dev {
+ tls internal
+ reverse_proxy github-stub:8000
+}
# bobbin (read appview / xrpc)
bobbin.tngl.boltless.dev {
tls internal
diff --git a/localinfra/github-stub.Dockerfile b/localinfra/github-stub.Dockerfile
new file mode 100644
index 000000000..e8697be3a
--- /dev/null
+++ b/localinfra/github-stub.Dockerfile
@@ -0,0 +1,17 @@
+FROM python:3.12-alpine
+
+RUN apk add --no-cache git
+
+WORKDIR /app
+COPY localinfra/github-stub /app/github-stub
+
+ENV PORT=8000 \
+ GITHUB_CLIENT_ID=ghstub-client-id \
+ GITHUB_CLIENT_SECRET=ghstub-client-secret \
+ GITHUB_APP_SLUG=tangled-dev-stub \
+ REPOS_DIR=/data/repos \
+ PYTHONUNBUFFERED=1
+
+EXPOSE 8000
+
+CMD ["python3", "/app/github-stub/server.py"]
diff --git a/localinfra/github-stub/__init__.py b/localinfra/github-stub/__init__.py
new file mode 100644
index 000000000..2cc75fcdf
--- /dev/null
+++ b/localinfra/github-stub/__init__.py
@@ -0,0 +1 @@
+"""Fake github.com, api.github.com and their git remotes, for local e2e runs."""
diff --git a/localinfra/github-stub/__main__.py b/localinfra/github-stub/__main__.py
new file mode 100644
index 000000000..2e660a801
--- /dev/null
+++ b/localinfra/github-stub/__main__.py
@@ -0,0 +1,4 @@
+from .server import run
+
+if __name__ == "__main__":
+ run()
diff --git a/localinfra/github-stub/server.py b/localinfra/github-stub/server.py
new file mode 100644
index 000000000..84336c331
--- /dev/null
+++ b/localinfra/github-stub/server.py
@@ -0,0 +1,616 @@
+#!/usr/bin/env python3
+import json
+import os
+import re
+import struct
+import subprocess
+import sys
+import tempfile
+import urllib.parse
+import zlib
+from datetime import datetime, timedelta, timezone
+from http.server import ThreadingHTTPServer, BaseHTTPRequestHandler
+
+PORT = int(os.environ.get("PORT", "8000"))
+CLIENT_ID = os.environ.get("GITHUB_CLIENT_ID", "ghstub-client-id")
+CLIENT_SECRET = os.environ.get("GITHUB_CLIENT_SECRET", "ghstub-client-secret")
+APP_SLUG = os.environ.get("GITHUB_APP_SLUG", "tangled-dev-stub")
+OBJECT_FORMAT = os.environ.get("GITHUB_STUB_OBJECT_FORMAT", "sha1")
+REPOS_DIR = os.environ.get("REPOS_DIR", os.path.join(tempfile.gettempdir(), "github-stub-repos"))
+
+PUBLIC_ORIGIN = os.environ.get("GITHUB_PUBLIC_ORIGIN", "https://github.com").rstrip("/")
+API_ORIGIN = os.environ.get("GITHUB_API_ORIGIN", "https://api.github.com").rstrip("/")
+AVATARS_ORIGIN = os.environ.get("GITHUB_AVATARS_ORIGIN", "https://avatars.githubusercontent.com").rstrip("/")
+
+OWNER = {
+ "login": "octostub",
+ "id": 10001,
+ "avatar_url": f"{AVATARS_ORIGIN}/u/10001",
+ "bio": "stub bio for the import flow",
+ "blog": "https://octostub.example.com",
+}
+
+# sha256 exercises knot2 object-format negotiation locally
+SEED = [
+ (
+ "hello",
+ "Hello world stub repository",
+ {
+ "README.md": "# Hello\n\nHello world stub repository for Tangled import testing.\n",
+ "hello.txt": "Hello from octostub/hello!\n",
+ },
+ "sha1",
+ ),
+ (
+ "world",
+ "World stub repository for bulk import",
+ {"README.md": "# World\n\nWorld stub repository for bulk import testing.\n"},
+ "sha1",
+ ),
+ (
+ "tangled-demo",
+ "Tangled demo repository",
+ {"README.md": "# Tangled Demo\n\nDemo repository for tangled import.\n"},
+ "sha1",
+ ),
+ (
+ "fresh",
+ "Fresh stub repository, never imported before",
+ {
+ "README.md": "# Fresh\n\nA repository nobody has imported yet.\n",
+ "src/lib.rs": "pub fn fresh() -> &'static str {\n \"fresh\"\n}\n",
+ },
+ "sha256",
+ ),
+ (
+ "second",
+ "Second fresh stub repository, for bulk import",
+ {
+ "README.md": "# Second\n\nThe second repository in a bulk import.\n",
+ "notes.md": "two\n",
+ },
+ "sha1",
+ ),
+ (
+ "toolkit",
+ "Small tools, for the bulk import",
+ {
+ "README.md": "# Toolkit\n\nTools nobody has imported yet.\n",
+ "tools.md": "hammer\n",
+ },
+ "sha1",
+ ),
+ (
+ "notes",
+ "Scratch notes, also unimported",
+ {"README.md": "# Notes\n\nNothing to see here yet.\n", "today.md": "one\n"},
+ "sha1",
+ ),
+]
+
+
+REPO_METADATA = {
+ "hello": {"language": "TypeScript", "stars": 12, "issues": 3, "forks": 2, "size_kb": 21504, "days_ago": 2},
+ "world": {"language": "Rust", "stars": 340, "issues": 7, "forks": 41, "size_kb": 512, "days_ago": 5},
+ "tangled-demo": {"language": "Go", "stars": 88, "issues": 1, "forks": 9, "size_kb": 330, "hours_ago": 3},
+ "fresh": {"language": "Rust", "stars": 0, "issues": 0, "forks": 1, "size_kb": 64, "days_ago": 1},
+ "second": {"language": "Python", "stars": 2048, "issues": 22, "forks": 190, "size_kb": 20480, "days_ago": 200},
+ "toolkit": {"language": "Shell", "stars": 5, "issues": 0, "forks": 0, "size_kb": 88, "days_ago": 3},
+ "notes": {"language": None, "stars": 1, "issues": 0, "forks": 0, "size_kb": 16, "days_ago": 45},
+}
+
+
+def _iso_age(**kwargs) -> str:
+ return (datetime.now(timezone.utc) - timedelta(**kwargs)).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+
+def repository(name: str, description: str, files: dict, object_format: str = "sha1") -> dict:
+ meta = REPO_METADATA[name]
+ pushed_at = _iso_age(hours=meta.get("hours_ago", 0), days=meta.get("days_ago", 0))
+ return {
+ "id": 101 + [seed[0] for seed in SEED].index(name),
+ "name": name,
+ "full_name": f"octostub/{name}",
+ "owner": OWNER,
+ "html_url": f"{PUBLIC_ORIGIN}/octostub/{name}",
+ "description": description,
+ "clone_url": f"{PUBLIC_ORIGIN}/octostub/{name}.git",
+ "default_branch": "main",
+ "visibility": "public",
+ "private": False,
+ "language": meta["language"],
+ "stargazers_count": meta["stars"],
+ "open_issues_count": meta["issues"],
+ "forks_count": meta["forks"],
+ "size": meta["size_kb"],
+ "pushed_at": pushed_at,
+ "updated_at": pushed_at,
+ "files": files,
+ "object_format": object_format,
+ }
+
+
+REPOSITORIES = [repository(*seed) for seed in SEED]
+
+STUB_TOKEN_PREFIX = "ghstub-token-"
+
+
+def email_for(token: str) -> str:
+ """The caller's address. Deliberi refuses one address for two accounts, so every
+ authorization gets its own: the token is derived from the code minted per connect."""
+ match = re.fullmatch(STUB_TOKEN_PREFIX + r"([0-9a-f]{16})", token)
+ suffix = f"+{match.group(1)[:8]}" if match else ""
+ return f"octostub{suffix}@octostub.dev"
+
+
+GITHUB_EMAILS = [
+ {"email": "octostub@octostub.dev", "primary": True, "verified": True, "visibility": "public"},
+ {
+ "email": "octostub-secondary@octostub.dev",
+ "primary": False,
+ "verified": False,
+ "visibility": "public",
+ },
+ {
+ "email": "octostub-alt@octostub.dev",
+ "primary": False,
+ "verified": True,
+ "visibility": "public",
+ },
+]
+
+
+def _clean_repo(r: dict) -> dict:
+ return {k: v for k, v in r.items() if k not in ("files", "object_format")}
+
+
+def make_png(width=16, height=16, color=(120, 80, 200)) -> bytes:
+ png = b"\x89PNG\r\n\x1a\n"
+ ihdr_data = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
+ ihdr_crc = struct.pack(">I", zlib.crc32(b"IHDR" + ihdr_data) & 0xffffffff)
+ png += struct.pack(">I", len(ihdr_data)) + b"IHDR" + ihdr_data + ihdr_crc
+
+ raw = bytearray()
+ for _ in range(height):
+ raw.append(0)
+ for _ in range(width):
+ raw.extend(color)
+
+ compressed = zlib.compress(bytes(raw))
+ idat_crc = struct.pack(">I", zlib.crc32(b"IDAT" + compressed) & 0xffffffff)
+ png += struct.pack(">I", len(compressed)) + b"IDAT" + compressed + idat_crc
+
+ iend_crc = struct.pack(">I", zlib.crc32(b"IEND") & 0xffffffff)
+ png += struct.pack(">I", 0) + b"IEND" + iend_crc
+ return png
+
+
+AVATAR_PNG = make_png()
+
+
+def seed_repositories():
+ os.makedirs(REPOS_DIR, exist_ok=True)
+ for repo_info in REPOSITORIES:
+ full_name = repo_info["full_name"]
+ bare_path = os.path.join(REPOS_DIR, f"{full_name}.git")
+ if os.path.exists(bare_path):
+ continue
+ os.makedirs(os.path.dirname(bare_path), exist_ok=True)
+ fmt = repo_info.get("object_format") or OBJECT_FORMAT
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ subprocess.run(
+ ["git", "init", f"--object-format={fmt}", tmp_dir],
+ check=True,
+ capture_output=True,
+ )
+ subprocess.run(["git", "-C", tmp_dir, "checkout", "-B", "main"], check=True, capture_output=True)
+ for fname, content in repo_info["files"].items():
+ fpath = os.path.join(tmp_dir, fname)
+ os.makedirs(os.path.dirname(fpath), exist_ok=True)
+ with open(fpath, "w") as f:
+ f.write(content)
+ subprocess.run(["git", "-C", tmp_dir, "config", "user.name", "Octo Stub"], check=True, capture_output=True)
+ subprocess.run(["git", "-C", tmp_dir, "config", "user.email", "octostub@octostub.dev"], check=True, capture_output=True)
+ subprocess.run(["git", "-C", tmp_dir, "add", "."], check=True, capture_output=True)
+ subprocess.run(["git", "-C", tmp_dir, "commit", "-m", "Initial commit"], check=True, capture_output=True)
+ subprocess.run(["git", "clone", "--bare", tmp_dir, bare_path], check=True, capture_output=True)
+ subprocess.run(["git", "-C", bare_path, "symbolic-ref", "HEAD", "refs/heads/main"], check=True, capture_output=True)
+ subprocess.run(["git", "-C", bare_path, "config", "http.receivepack", "false"], check=True, capture_output=True)
+ desc_path = os.path.join(bare_path, "description")
+ with open(desc_path, "w") as f:
+ f.write(repo_info["description"] + "\n")
+ print(f"Seeded bare repository ({fmt}): {bare_path}", flush=True)
+
+
+class GitHubStubHandler(BaseHTTPRequestHandler):
+ protocol_version = "HTTP/1.1"
+
+ def log_message(self, format, *args):
+ sys.stderr.write(f"[{self.log_date_time_string()}] {self.address_string()} {format % args}\n")
+ sys.stderr.flush()
+
+ def read_body(self) -> bytes:
+ if self.headers.get("Transfer-Encoding", "").lower() == "chunked":
+ chunks = []
+ while True:
+ line = self.rfile.readline().strip()
+ if not line:
+ break
+ chunk_len = int(line.split(b";")[0], 16)
+ if chunk_len == 0:
+ while self.rfile.readline().strip():
+ pass
+ break
+ chunk = self.rfile.read(chunk_len)
+ chunks.append(chunk)
+ self.rfile.readline()
+ return b"".join(chunks)
+ length = int(self.headers.get("Content-Length", 0))
+ return self.rfile.read(length) if length > 0 else b""
+
+ def send_json(self, status: int, data: any, link: str = ""):
+ body = json.dumps(data, indent=2).encode("utf-8")
+ self.send_response(status)
+ self.send_header("Content-Type", "application/json; charset=utf-8")
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("Cache-Control", "no-store")
+ if link:
+ self.send_header("Link", link)
+ self.end_headers()
+ self.wfile.write(body)
+
+ def page_link(self, path: str, query: dict, page: int, per_page: int, total: int) -> str:
+ if page * per_page >= total:
+ return ""
+ params = {k: v[0] for k, v in query.items()}
+ params.update({"page": str(page + 1), "per_page": str(per_page)})
+ nxt = urllib.parse.urlencode(params)
+ return f'<{API_ORIGIN}{path}?{nxt}>; rel="next"'
+
+ def send_avatar(self):
+ self.send_response(200)
+ self.send_header("Content-Type", "image/png")
+ self.send_header("Content-Length", str(len(AVATAR_PNG)))
+ self.send_header("Cache-Control", "public, max-age=86400")
+ self.end_headers()
+ self.wfile.write(AVATAR_PNG)
+
+ def do_GET(self):
+ parsed = urllib.parse.urlparse(self.path)
+ path = parsed.path.rstrip("/")
+ if not path:
+ path = "/"
+ query = urllib.parse.parse_qs(parsed.query)
+
+ if path.startswith("/u/") or path.startswith("/avatar") or path.startswith("/avatars/") or path == "/user/avatar":
+ self.send_avatar()
+ return
+
+ git_refs_match = re.match(r"^/([^/]+)/([^/]+?)(?:\.git)?/info/refs$", parsed.path)
+ if git_refs_match:
+ owner, repo_name = git_refs_match.group(1), git_refs_match.group(2)
+ service = query.get("service", [""])[0]
+ if service != "git-upload-pack":
+ self.send_error(400, "Unsupported git service")
+ return
+ repo_path = os.path.join(REPOS_DIR, owner, f"{repo_name}.git")
+ if not os.path.exists(repo_path):
+ self.send_error(404, "Repository not found")
+ return
+ env = os.environ.copy()
+ proto = self.headers.get("Git-Protocol")
+ if proto:
+ env["GIT_PROTOCOL"] = proto
+ env["GIT_CONFIG_NOSYSTEM"] = "1"
+ proc = subprocess.run(
+ ["git", "upload-pack", "--stateless-rpc", "--advertise-refs", repo_path],
+ capture_output=True,
+ env=env
+ )
+ if proc.returncode != 0:
+ self.send_error(500, f"git upload-pack failed: {proc.stderr.decode('utf-8', errors='replace')}")
+ return
+ body = b"001e# service=git-upload-pack\n0000" + proc.stdout
+ self.send_response(200)
+ self.send_header("Content-Type", "application/x-git-upload-pack-advertisement")
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+ self.send_header("Pragma", "no-cache")
+ self.end_headers()
+ self.wfile.write(body)
+ return
+
+ if path == "/login/oauth/authorize":
+ redirect_uri = query.get("redirect_uri", [""])[0]
+ state = query.get("state", [""])[0]
+ code = "ghstub-" + os.urandom(8).hex()
+ if not redirect_uri:
+ self.send_error(400, "Missing redirect_uri")
+ return
+ sep = "&" if "?" in redirect_uri else "?"
+ dest = f"{redirect_uri}{sep}code={code}&state={urllib.parse.quote(state)}"
+ self.send_response(302)
+ self.send_header("Location", dest)
+ self.send_header("Cache-Control", "no-store")
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
+
+ if re.match(r"^/apps/[^/]+/installations/new$", path):
+ setup = os.environ.get(
+ "GITHUB_APP_SETUP_URL",
+ "http://127.0.0.1:5174/_internal/github/install/callback",
+ )
+ sep = "&" if "?" in setup else "?"
+ state = query.get("state", [""])[0]
+ dest = f"{setup}{sep}installation_id=1&setup_action=install"
+ if state:
+ dest += f"&state={urllib.parse.quote(state)}"
+ self.send_response(302)
+ self.send_header("Location", dest)
+ self.send_header("Cache-Control", "no-store")
+ self.send_header("Content-Length", "0")
+ self.end_headers()
+ return
+
+ if path == "/stub/installed":
+ html = (
+ b"GitHub App Installed"
+ b"GitHub App Installed
"
+ b"The Tangled GitHub App has been installed on your stub account.
"
+ b""
+ )
+ self.send_response(200)
+ self.send_header("Content-Type", "text/html; charset=utf-8")
+ self.send_header("Content-Length", str(len(html)))
+ self.end_headers()
+ self.wfile.write(html)
+ return
+
+ if path == "/user":
+ token = self.headers.get("Authorization", "").removeprefix("Bearer ").strip()
+ user_data = {
+ "login": "octostub",
+ "id": 10001,
+ "node_id": "MDQ6VXNlcjEwMDAx",
+ "avatar_url": f"{AVATARS_ORIGIN}/u/10001",
+ "gravatar_id": "",
+ "url": f"{API_ORIGIN}/users/octostub",
+ "html_url": f"{PUBLIC_ORIGIN}/octostub",
+ "type": "User",
+ "site_admin": False,
+ "name": "Octo Stub",
+ "company": None,
+ "blog": "https://octostub.example.com",
+ "location": None,
+ "email": email_for(token),
+ "hireable": None,
+ "bio": "Stub GitHub user for local development",
+ "public_repos": len(REPOSITORIES),
+ "public_gists": 0,
+ "followers": 0,
+ "following": 0,
+ "created_at": "2024-01-01T00:00:00Z",
+ "updated_at": "2024-01-01T00:00:00Z"
+ }
+ self.send_json(200, user_data)
+ return
+
+ if path == "/user/emails":
+ token = self.headers.get("Authorization", "").removeprefix("Bearer ").strip()
+ address = email_for(token)
+ local_suffix = address[len("octostub") : address.index("@")]
+ self.send_json(
+ 200,
+ [
+ {**row, "email": row["email"].replace("@octostub.dev", "").replace("octostub", "octostub" + local_suffix) + "@octostub.dev"}
+ for row in GITHUB_EMAILS
+ ],
+ )
+ return
+
+ if path == "/user/keys":
+ keys = [
+ {
+ "id": 1,
+ "key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOr0s3gZf7W/8vV/EXAMPLEEXAMPLEEXAMPLEEXAMPLE octostub@octostub.dev",
+ "title": "Stub Key"
+ }
+ ]
+ self.send_json(200, keys)
+ return
+
+ if path == "/user/installations":
+ installations_data = {
+ "total_count": 1,
+ "installations": [
+ {
+ "id": 1,
+ "account": {
+ "login": "octostub",
+ "id": 10001,
+ "type": "User",
+ "avatar_url": f"{AVATARS_ORIGIN}/u/10001"
+ },
+ "app_id": 1,
+ "app_slug": APP_SLUG,
+ "target_id": 10001,
+ "target_type": "User",
+ "permissions": {
+ "metadata": "read",
+ "contents": "read"
+ },
+ "events": [],
+ "repository_selection": "all"
+ }
+ ]
+ }
+ self.send_json(200, installations_data)
+ return
+
+ inst_repos_match = re.match(r"^/user/installations/(\d+)/repositories$", path)
+ if inst_repos_match:
+ clean_repos = [_clean_repo(r) for r in REPOSITORIES]
+ page = max(1, int(query.get("page", ["1"])[0]))
+ per_page = max(1, min(100, int(query.get("per_page", ["30"])[0])))
+ self.send_json(200, {
+ "total_count": len(clean_repos),
+ "repositories": clean_repos[(page - 1) * per_page : page * per_page]
+ }, self.page_link(path, query, page, per_page, len(clean_repos)))
+ return
+
+ if path == "/user/repos":
+ visibility = query.get("visibility", [""])[0]
+ affiliation = query.get("affiliation", ["owner,collaborator,organization_member"])[0]
+ page = max(1, int(query.get("page", ["1"])[0]))
+ per_page = max(1, min(100, int(query.get("per_page", ["30"])[0])))
+ owned = "owner" in {a for a in affiliation.split(",") if a}
+ rows = [] if not owned else [
+ _clean_repo(r)
+ for r in REPOSITORIES
+ if not visibility or r.get("visibility") == visibility
+ ]
+ self.send_json(200, rows[(page - 1) * per_page : page * per_page],
+ self.page_link(path, query, page, per_page, len(rows)))
+ return
+
+ users_repos_match = re.match(r"^/users/([^/]+)/repos$", path)
+ if users_repos_match:
+ login = users_repos_match.group(1)
+ page = max(1, int(query.get("page", ["1"])[0]))
+ per_page = max(1, min(100, int(query.get("per_page", ["30"])[0])))
+ rows = [
+ _clean_repo(r)
+ for r in REPOSITORIES
+ if r.get("visibility") == "public" and r.get("full_name", "").startswith(f"{login}/")
+ ]
+ self.send_json(200, rows[(page - 1) * per_page : page * per_page],
+ self.page_link(path, query, page, per_page, len(rows)))
+ return
+
+ repo_match = re.match(r"^/repos/([^/]+)/([^/]+)$", path)
+ if repo_match:
+ owner, name = repo_match.group(1), repo_match.group(2)
+ full_name = f"{owner}/{name}"
+ for r in REPOSITORIES:
+ if r["full_name"] == full_name:
+ self.send_json(200, _clean_repo(r))
+ return
+ self.send_json(404, {"message": "Not Found"})
+ return
+
+ if path == "/":
+ self.send_json(200, {
+ "status": "ok",
+ "service": "github-stub",
+ "client_id": CLIENT_ID,
+ "app_slug": APP_SLUG,
+ "repositories": [r["full_name"] for r in REPOSITORIES]
+ })
+ return
+
+ self.send_json(404, {"message": f"Not Found: {path}"})
+
+ def do_POST(self):
+ parsed = urllib.parse.urlparse(self.path)
+ path = parsed.path.rstrip("/")
+ if not path:
+ path = "/"
+
+ git_pack_match = re.match(r"^/([^/]+)/([^/]+?)(?:\.git)?/git-upload-pack$", parsed.path)
+ if git_pack_match:
+ owner, repo_name = git_pack_match.group(1), git_pack_match.group(2)
+ repo_path = os.path.join(REPOS_DIR, owner, f"{repo_name}.git")
+ if not os.path.exists(repo_path):
+ self.send_error(404, "Repository not found")
+ return
+
+ body = self.read_body()
+ content_encoding = self.headers.get("Content-Encoding", "").lower()
+ if content_encoding == "gzip":
+ body = zlib.decompress(body, 16 + zlib.MAX_WBITS)
+ elif content_encoding == "deflate":
+ body = zlib.decompress(body)
+
+ env = os.environ.copy()
+ proto = self.headers.get("Git-Protocol")
+ if proto:
+ env["GIT_PROTOCOL"] = proto
+ env["GIT_CONFIG_NOSYSTEM"] = "1"
+
+ proc = subprocess.Popen(
+ ["git", "upload-pack", "--stateless-rpc", repo_path],
+ stdin=subprocess.PIPE,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ env=env
+ )
+ stdout, stderr = proc.communicate(input=body)
+ if proc.returncode != 0:
+ self.send_error(500, f"git upload-pack failed: {stderr.decode('utf-8', errors='replace')}")
+ return
+
+ self.send_response(200)
+ self.send_header("Content-Type", "application/x-git-upload-pack-result")
+ self.send_header("Content-Length", str(len(stdout)))
+ self.send_header("Cache-Control", "no-cache, no-store, must-revalidate")
+ self.send_header("Pragma", "no-cache")
+ self.end_headers()
+ self.wfile.write(stdout)
+ return
+
+ if path == "/login/oauth/access_token":
+ raw_body = self.read_body()
+ content_type = self.headers.get("Content-Type", "").lower()
+
+ params = {}
+ if "application/json" in content_type:
+ try:
+ params = json.loads(raw_body.decode("utf-8"))
+ except Exception:
+ pass
+ else:
+ try:
+ parsed_qs = urllib.parse.parse_qs(raw_body.decode("utf-8"))
+ params = {k: v[0] for k, v in parsed_qs.items()}
+ except Exception:
+ pass
+
+ token_data = {
+ "access_token": STUB_TOKEN_PREFIX
+ + (str(params.get("code", "")).removeprefix("ghstub-") or "0123456789abcdef")[:16],
+ "token_type": "bearer",
+ "scope": "repo,read:user,user:email"
+ }
+
+ accept = self.headers.get("Accept", "").lower()
+ if "application/json" in accept or "application/json" in content_type:
+ self.send_json(200, token_data)
+ else:
+ body = urllib.parse.urlencode(token_data).encode("utf-8")
+ self.send_response(200)
+ self.send_header("Content-Type", "application/x-www-form-urlencoded")
+ self.send_header("Content-Length", str(len(body)))
+ self.send_header("Cache-Control", "no-store")
+ self.end_headers()
+ self.wfile.write(body)
+ return
+
+ self.send_json(404, {"message": f"Not Found: {path}"})
+
+
+def run():
+ print(f"Starting GitHub Stub on port {PORT}...", flush=True)
+ seed_repositories()
+ server = ThreadingHTTPServer(("0.0.0.0", PORT), GitHubStubHandler)
+ print(f"GitHub Stub listening on 0.0.0.0:{PORT}", flush=True)
+ try:
+ server.serve_forever()
+ except KeyboardInterrupt:
+ pass
+ finally:
+ server.server_close()
+
+
+if __name__ == "__main__":
+ run()
diff --git a/localinfra/migrator.Dockerfile b/localinfra/migrator.Dockerfile
new file mode 100644
index 000000000..a5b30a428
--- /dev/null
+++ b/localinfra/migrator.Dockerfile
@@ -0,0 +1,35 @@
+# Not for production: the migrator serves its did:web over plain http and trusts caddy's ca.
+
+FROM golang:1.26-alpine AS builder
+
+RUN apk add --no-cache git build-base sqlite-dev
+
+ENV CGO_ENABLED=1
+ENV GOCACHE=/go/cache
+ENV GOMODCACHE=/go/mod
+
+WORKDIR /src
+
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/go/cache \
+ --mount=type=cache,target=/go/mod \
+ go mod download
+
+COPY . .
+RUN --mount=type=cache,target=/go/cache \
+ --mount=type=cache,target=/go/mod \
+ go build -tags libsqlite3 -o /out/migrator ./cmd/migrator
+
+FROM alpine:3.24
+
+# git-lfs is not optional: the worker pushes lfs objects before refs, and a
+# mirror that lands without its objects is the failure this service avoids
+RUN apk add --no-cache ca-certificates git git-lfs sqlite-libs tini
+
+COPY --from=builder /out/migrator /usr/local/bin/migrator
+
+VOLUME /var/lib/migrator
+EXPOSE 6767
+
+ENTRYPOINT ["/sbin/tini", "--"]
+CMD ["sh", "-c", "if [ -f /usr/local/share/ca-certificates/caddy.crt ]; then update-ca-certificates; fi && exec /usr/local/bin/migrator serve"]
diff --git a/localinfra/readme.md b/localinfra/readme.md
index e17c5f614..d7d43ae2a 100644
--- a/localinfra/readme.md
+++ b/localinfra/readme.md
@@ -37,24 +37,12 @@ To make that work:
## Setup
-1. Generate the dev CA from the repo root:
+1. Generate the dev CA and the local secrets from the repo root:
```bash
- mkdir -p localinfra/certs &&
- openssl req -x509 -newkey rsa:2048 \
- -keyout localinfra/certs/root.key \
- -out localinfra/certs/root.crt \
- -days 3650 -nodes \
- -subj "/CN=Tangled Dev CA" \
- -addext "basicConstraints=critical,CA:TRUE,pathlen:1" \
- -addext "keyUsage=critical,keyCertSign,cRLSign" \
- -addext "nameConstraints=critical,permitted;DNS:tngl.boltless.dev"
-
- test -f localinfra/certs/service-auth.env || (
- umask 077
- nix develop .#default -c go run ./cmd/knotmirror generate-service-auth-key \
- > localinfra/certs/service-auth.env
- )
+ ./localinfra/scripts/make-certs.sh
```
+ It writes root.crt plus the env files the services read (github.env, migrator.env,
+ service-auth.env) and skips whatever already exists.
2. Trust generated `localinfra/certs/root.crt` in your system's trust store.
- For example in MacOS, run
```bash
diff --git a/localinfra/scripts/add-accounts.sh b/localinfra/scripts/add-accounts.sh
new file mode 100644
index 000000000..cf992bd45
--- /dev/null
+++ b/localinfra/scripts/add-accounts.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+# pds account creation only; safe to run against a live rig without touching fixtures
+set -eu
+
+. /scripts/lib.sh
+
+USERS="${*:-${USERS:-}}"
+[ -n "$USERS" ] || fail "usage: add-accounts.sh name [name ...]" \
+ " or set USERS to a space-separated list."
+
+: "${PDS_HOSTNAME:?PDS_HOSTNAME must be set}"
+: "${PDS_ADMIN_PASSWORD:?PDS_ADMIN_PASSWORD must be set}"
+
+for u in $USERS; do
+ handle="${u}.${PDS_HOSTNAME}"
+ email="${u}@${PDS_HOSTNAME}"
+ did=$(resolve_handle "$handle")
+ if [ -n "$did" ]; then
+ printf '[skip] %s = %s\n' "$handle" "$did" >&2
+ else
+ invite=$(curl -fsS -u "admin:${PDS_ADMIN_PASSWORD}" \
+ -H "Content-Type: application/json" \
+ -d '{"useCount":1}' \
+ "${PDS_URL}/xrpc/com.atproto.server.createInviteCode" | jq -er '.code')
+
+ result=$(curl -fsS \
+ -H "Content-Type: application/json" \
+ -d "{\"email\":\"${email}\",\"handle\":\"${handle}\",\"password\":\"${PASSWORD}\",\"inviteCode\":\"${invite}\"}" \
+ "${PDS_URL}/xrpc/com.atproto.server.createAccount")
+
+ did=$(printf '%s\n' "$result" | jq -er '.did')
+ printf '[create] %s = %s (password: %s)\n' "$handle" "$did" "$PASSWORD" >&2
+ fi
+ printf '%s\t%s\n' "$u" "$did"
+done
diff --git a/localinfra/scripts/init-accounts.sh b/localinfra/scripts/init-accounts.sh
index cadc49615..ce902120a 100644
--- a/localinfra/scripts/init-accounts.sh
+++ b/localinfra/scripts/init-accounts.sh
@@ -16,38 +16,40 @@ VERIFIED_EMAILS_FILE="${SHARED_DIR}/verified-emails.jsonl"
VERIFIED_EMAILS_TMP="${VERIFIED_EMAILS_FILE}.tmp"
TRANQUIL_PASSWORD="${TRANQUIL_PASSWORD:-Tangled-Dev9}"
-# --- helpers ---
-
-# ensure_account USERNAME → DID on stdout. Creates account if missing.
-ensure_account() {
- username="$1"
- handle="${username}.${PDS_HOSTNAME}"
- email="${username}@${PDS_HOSTNAME}"
+mkdir -p "$SHARED_DIR"
+: > "$VERIFIED_EMAILS_TMP"
+ACCOUNTS_FILE="${SHARED_DIR}/accounts.tsv"
+add-accounts.sh $USERS > "$ACCOUNTS_FILE"
+cat "$ACCOUNTS_FILE" >&2
- did=$(resolve_handle "$handle")
- if [ -n "$did" ]; then
- printf '[skip] %s = %s\n' "$handle" "$did" >&2
- printf '%s\n' "$did"
- return 0
+OWNER_DID=""
+SYSTEM_DID=""
+while IFS="$(printf '\t')" read -r u did; do
+ [ -n "$u" ] || continue
+ jq -cn \
+ --arg did "$did" \
+ --arg email "${u}@${PDS_HOSTNAME}" \
+ '{did: $did, email: $email}' >> "$VERIFIED_EMAILS_TMP"
+ if [ "$u" = "$OWNER_USER" ]; then
+ OWNER_DID="$did"
fi
+ if [ "$u" = "$SYSTEM_USER" ]; then
+ SYSTEM_DID="$did"
+ fi
+done < "$ACCOUNTS_FILE"
- invite=$(curl -fsS -u "admin:${PDS_ADMIN_PASSWORD}" \
- -H "Content-Type: application/json" \
- -d '{"useCount":1}' \
- "${PDS_URL}/xrpc/com.atproto.server.createInviteCode" | jq -er '.code')
-
- result=$(curl -fsS \
- -H "Content-Type: application/json" \
- -d "{\"email\":\"${email}\",\"handle\":\"${handle}\",\"password\":\"${PASSWORD}\",\"inviteCode\":\"${invite}\"}" \
- "${PDS_URL}/xrpc/com.atproto.server.createAccount")
+[ -n "$OWNER_DID" ] || { printf 'OWNER_USER %s not in USERS list\n' "$OWNER_USER" >&2; exit 1; }
+[ -n "$SYSTEM_DID" ] || { printf 'SYSTEM_USER %s not in USERS list\n' "$SYSTEM_USER" >&2; exit 1; }
- did=$(printf '%s\n' "$result" | jq -er '.did')
- printf '[create] %s = %s (password: %s)\n' "$handle" "$did" "$PASSWORD" >&2
- printf '%s\n' "$did"
-}
+mv "$VERIFIED_EMAILS_TMP" "$VERIFIED_EMAILS_FILE"
+printf '[emails] wrote verified committer fixtures for %s\n' "$USERS" >&2
+printf '%s' "$OWNER_DID" > "${SHARED_DIR}/owner-did"
+printf '[owner] %s → %s/owner-did\n' "$OWNER_USER" "$SHARED_DIR" >&2
+printf '%s' "$SYSTEM_DID" > "${SHARED_DIR}/system-did"
+printf '[system] %s → %s/system-did\n' "$SYSTEM_USER" "$SHARED_DIR" >&2
-# ensure_tranquil_account USERNAME → DID on stdout. same as ensure_account,
-# but against tranquil's own URL/hostname, password, and no invite code.
+# tranquil-pds account helper: same shape as add-accounts.sh, but against
+# tranquil's own URL/hostname, password, and no invite code.
ensure_tranquil_account() {
username="$1"
handle="${username}.${TRANQUIL_HOSTNAME}"
@@ -71,35 +73,6 @@ ensure_tranquil_account() {
printf '%s\n' "$did"
}
-# ensure accounts
-mkdir -p "$SHARED_DIR"
-: > "$VERIFIED_EMAILS_TMP"
-OWNER_DID=""
-SYSTEM_DID=""
-for u in $USERS; do
- did=$(ensure_account "$u")
- jq -cn \
- --arg did "$did" \
- --arg email "${u}@${PDS_HOSTNAME}" \
- '{did: $did, email: $email}' >> "$VERIFIED_EMAILS_TMP"
- if [ "$u" = "$OWNER_USER" ]; then
- OWNER_DID="$did"
- fi
- if [ "$u" = "$SYSTEM_USER" ]; then
- SYSTEM_DID="$did"
- fi
-done
-
-[ -n "$OWNER_DID" ] || { printf 'OWNER_USER %s not in USERS list\n' "$OWNER_USER" >&2; exit 1; }
-[ -n "$SYSTEM_DID" ] || { printf 'SYSTEM_USER %s not in USERS list\n' "$SYSTEM_USER" >&2; exit 1; }
-
-mv "$VERIFIED_EMAILS_TMP" "$VERIFIED_EMAILS_FILE"
-printf '[emails] wrote verified committer fixtures for %s\n' "$USERS" >&2
-printf '%s' "$OWNER_DID" > "${SHARED_DIR}/owner-did"
-printf '[owner] %s → %s/owner-did\n' "$OWNER_USER" "$SHARED_DIR" >&2
-printf '%s' "$SYSTEM_DID" > "${SHARED_DIR}/system-did"
-printf '[system] %s → %s/system-did\n' "$SYSTEM_USER" "$SHARED_DIR" >&2
-
# label definitions (under SYSTEM_DID)
put_record "at://$SYSTEM_DID/sh.tangled.label.definition/wontfix" "$(cat < "$CERTS/github.env" </dev/null 2>&1; then
+ go run ./cmd/knotmirror generate-service-auth-key
+ else
+ nix develop .#default -c go run ./cmd/knotmirror generate-service-auth-key
+ fi
+}
+
+# openssl cannot make an atproto multibase secp256k1 key
+if [ ! -s "$CERTS/migrator.env" ]; then
+ gen_key | sed 's/^MIRROR_SERVICE_PRIVATE_KEY=/MIGRATOR_PRIVATE_KEY=/' > "$CERTS/migrator.env"
+ {
+ printf 'MIGRATOR_MASTER_KEY=%s\n' "$(head -c 32 /dev/urandom | base64)"
+ printf 'MIGRATOR_API_TOKEN=%s\n' "$(head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n')"
+ printf 'MIGRATOR_URL=https://migrator.tngl.boltless.dev\n'
+ printf 'MIGRATOR_DID=did:web:migrator.tngl.boltless.dev\n'
+ printf 'MIGRATOR_PLC_URL=https://plc.tngl.boltless.dev\n'
+ printf 'MIGRATOR_GITHUB_HOST=github.tngl.boltless.dev\n'
+ } >> "$CERTS/migrator.env"
+fi
+
+if [ ! -s "$CERTS/service-auth.env" ]; then
+ gen_key > "$CERTS/service-auth.env"
+fi
+
+printf 'certs and keys are ready in %s\n' "$CERTS"