From 8d955862a7c78b9e879f123afcc2923ca532e4ba Mon Sep 17 00:00:00 2001 From: Owais Jamil Date: Mon, 23 Mar 2026 21:31:00 -0500 Subject: [PATCH] feat: migrate to charm log * add tests and deployment/qa guides --- docs/api/deploy.md | 100 ++++++++++++++++++++ docs/api/tasks/phase-1-mvp.md | 20 ++-- docs/qa.md | 77 +++++++++++++++ packages/api/.dockerignore | 5 + packages/api/go.mod | 13 +++ packages/api/go.sum | 37 ++++++++ packages/api/internal/config/config.go | 2 + packages/api/internal/ingest/ingest_test.go | 4 + packages/api/internal/observability/log.go | 20 ++-- packages/api/internal/store/sql_store.go | 4 + packages/api/internal/store/store.go | 1 + packages/api/main.go | 30 ++++++ 12 files changed, 294 insertions(+), 19 deletions(-) create mode 100644 docs/api/deploy.md create mode 100644 docs/qa.md create mode 100644 packages/api/.dockerignore diff --git a/docs/api/deploy.md b/docs/api/deploy.md new file mode 100644 index 0000000..713cb20 --- /dev/null +++ b/docs/api/deploy.md @@ -0,0 +1,100 @@ +--- +title: "Deployment Guide" +updated: 2026-03-23 +--- + +# Railway Deployment Guide + +Deploy the Twister API and indexer as Railway services alongside the existing Tap instance. + +## Prerequisites + +- Railway project with Tap already deployed +- Turso database created with auth token +- GitHub repository connected to Railway + +## Service Layout + +| Service | Start Command | Health Check | Public | Port | +| ------- | ----------------- | -------------- | ------ | ---- | +| tap | (pre-existing) | `GET /health` | no | — | +| api | `twister api` | `GET /healthz` | yes | 8080 | +| indexer | `twister indexer` | `GET /health` | no | 9090 | + +All services use the same Docker image. Railway overrides `CMD` with the per-service start command. + +## Step 1 — Create Services + +In the Railway dashboard, create two new services from the same GitHub repo: + +1. **api** — set start command to `twister api` +2. **indexer** — set start command to `twister indexer` + +Both services build from `packages/api/Dockerfile`. + +## Step 2 — Set Environment Variables + +### Shared (set on both services) + +```sh +TURSO_DATABASE_URL=libsql://twister-prod-.turso.io +TURSO_AUTH_TOKEN= +LOG_LEVEL=info +LOG_FORMAT=json +``` + +### API only + +```sh +HTTP_BIND_ADDR=:8080 +SEARCH_DEFAULT_LIMIT=20 +SEARCH_MAX_LIMIT=100 +``` + +### Indexer only + +```sh +TAP_URL=wss://${{tap.RAILWAY_PRIVATE_DOMAIN}}/channel +TAP_AUTH_PASSWORD= +INDEXED_COLLECTIONS=sh.tangled.repo,sh.tangled.repo.issue,sh.tangled.repo.pull,sh.tangled.string,sh.tangled.actor.profile,sh.tangled.repo.issue.comment,sh.tangled.repo.pull.comment,sh.tangled.repo.issue.state,sh.tangled.repo.pull.status,sh.tangled.feed.star +INDEXER_HEALTH_ADDR=:9090 +``` + +Use `${{tap.RAILWAY_PRIVATE_DOMAIN}}` to reference Tap's internal hostname. This keeps traffic on Railway's private network. + +## Step 3 — Configure Health Checks + +In the Railway dashboard, configure per-service: + +- **api**: HTTP health check on path `/healthz`, port `8080` +- **indexer**: HTTP health check on path `/health`, port `9090` + +Railway uses these to gate deployment rollouts and restart unhealthy containers. + +## Step 4 — Configure Autodeploy + +Connect the GitHub repository in the Railway dashboard. Railway will build and deploy on every push to the configured branch. + +The Dockerfile uses multi-stage builds with `CGO_ENABLED=0` for a static binary on Alpine. + +## Step 5 — Deploy and Verify + +After the first deploy: + +1. Confirm API is healthy: `curl https:///healthz` +2. Confirm API readiness: `curl https:///readyz` +3. Check indexer health in Railway logs (health check on `:9090/health`) + +## Step 6 — Bootstrap Content + +Run graph backfill to populate initial content from seed users: + +```bash +twister backfill --seeds=docs/api/seeds.txt --max-hops=2 +``` + +Wait for Tap to finish historical sync, then verify search returns results: + +```bash +curl "https:///search?q=tangled" +``` diff --git a/docs/api/tasks/phase-1-mvp.md b/docs/api/tasks/phase-1-mvp.md index 03c7296..68699be 100644 --- a/docs/api/tasks/phase-1-mvp.md +++ b/docs/api/tasks/phase-1-mvp.md @@ -318,9 +318,9 @@ Ship a static site that doubles as public API documentation and a live search de A user can search Tangled content and read API docs from a public URL without installing anything. -## M6 — Railway Deployment +## M6 — Railway Deployment ✅ -refs: [specs/06-operations.md](../specs/06-operations.md) +refs: [specs/06-operations.md](../specs/06-operations.md), [deploy.md](../deploy.md) ### Goal @@ -336,21 +336,21 @@ Deploy the API and indexer as Railway services alongside Tap. ### Tasks -- [ ] Finalize Dockerfile (multi-stage, CGO_ENABLED=0, Alpine runtime) -- [ ] Create Railway services: +- [x] Finalize Dockerfile (multi-stage, CGO_ENABLED=0, Alpine runtime) +- [x] Create Railway services: - `api` — start command: `twister api` - `indexer` — start command: `twister indexer` -- [ ] Configure environment variables per service: +- [x] Configure environment variables per service: - Shared: `TURSO_DATABASE_URL`, `TURSO_AUTH_TOKEN`, `LOG_LEVEL`, `LOG_FORMAT` - API: `HTTP_BIND_ADDR`, `SEARCH_DEFAULT_LIMIT`, `SEARCH_MAX_LIMIT` - Indexer: `TAP_URL` (reference Tap service domain), `TAP_AUTH_PASSWORD`, `INDEXED_COLLECTIONS` -- [ ] Configure health checks: +- [x] Configure health checks: - API: HTTP check on `/healthz` port 8080 - Indexer: HTTP check on `/health` port 9090 -- [ ] Use Railway internal networking for indexer → Tap connection -- [ ] Connect GitHub repo for autodeploy -- [ ] Test graceful shutdown on redeploy (SIGTERM handling) -- [ ] Document deploy steps +- [x] Use Railway internal networking for indexer → Tap connection +- [x] Connect GitHub repo for autodeploy +- [x] Test graceful shutdown on redeploy (SIGTERM handling) +- [x] Document deploy steps ### Verification diff --git a/docs/qa.md b/docs/qa.md new file mode 100644 index 0000000..ea48e4d --- /dev/null +++ b/docs/qa.md @@ -0,0 +1,77 @@ +--- +title: "QA Checklist" +updated: 2026-03-23 +--- + +# QA Checklist + +## Ingestion (end-to-end) + +Walk a record through the full pipeline: Tap event → indexer → store → searchable. + +- [ ] Indexer connects to Tap via WebSocket and begins processing events +- [ ] Creating a tracked record on Tangled produces a row in `documents` +- [ ] Updating that record changes the existing row (new CID) +- [ ] Deleting that record tombstones the row (`deleted_at` set) +- [ ] Tombstoned documents do not appear in search results +- [ ] Identity events update the handle cache; new documents show resolved handles +- [ ] Unsupported collections are silently skipped (no errors logged) +- [ ] Connection drop triggers automatic reconnect and resumes from last cursor + +## Cursor durability + +- [ ] Kill the indexer mid-stream, restart — processing resumes without duplicating documents +- [ ] Redeploy the indexer — cursor is persisted before shutdown, no gap or replay + +## Backfill + +Run `twister backfill` against a small seed file and verify the discovery graph. + +- [ ] Seed file with known Tangled users produces a non-empty discovery graph +- [ ] `--max-hops 1` limits discovery to direct follows/collaborators only +- [ ] `--dry-run` logs the plan but does not call Tap mutation endpoints +- [ ] Already-tracked DIDs are reported and not re-submitted +- [ ] Re-running the same seeds is idempotent +- [ ] After backfill + Tap sync, search returns historical content that wasn't there before + +## Search API + +- [ ] `GET /search?q=` returns the expected repo as top result +- [ ] Searching by title keyword returns expected documents +- [ ] Searching by author handle returns their content +- [ ] `collection`, `type`, `author`, `repo` filters restrict results correctly +- [ ] Pagination: `offset=0&limit=5` then `offset=5&limit=5` return disjoint result sets +- [ ] Missing `q` param returns 400 with error JSON +- [ ] Unknown query param returns 400 +- [ ] `GET /documents/{id}` returns the full document; 404 for missing or tombstoned +- [ ] `GET /healthz` returns 200 +- [ ] `GET /readyz` returns 503 when DB is unreachable + +## Deployment (Railway) + +- [ ] API service healthy and routable at public URL +- [ ] Indexer service healthy on `:9090/health` +- [ ] A new Tangled record ingested post-deploy becomes searchable within seconds +- [ ] Redeploying the API preserves availability (health-check-gated rollout) +- [ ] Restarting the indexer does not lose sync position +- [ ] Environment variables match the documented set in `docs/api/deploy.md` + +## Mobile — Navigation & Shell + +- [ ] All five tabs render and switch without layout shift +- [ ] Tab-to-tab navigation preserves scroll position and component state +- [ ] Pages show skeleton loaders before data appears +- [ ] iOS and Android builds compile and launch via Capacitor + +## Mobile — Live Tangled Browsing + +- [ ] Repo detail page loads metadata from PDS + git data from knot +- [ ] README renders via markdown renderer +- [ ] File tree navigates directories; file viewer shows syntax-highlighted content +- [ ] Commit log paginates with cursor +- [ ] Profile page shows avatar, bio, and repos from PDS +- [ ] Issue list filters by state (open/closed); detail shows body + threaded comments +- [ ] PR list filters by status; detail shows source/target branches + comments +- [ ] Stale-while-revalidate: cached data shows immediately, refreshes in background +- [ ] Error states render correctly: 404, network failure, empty repo +- [ ] Slow network: skeleton → content transition is smooth (test with throttled devtools) diff --git a/packages/api/.dockerignore b/packages/api/.dockerignore new file mode 100644 index 0000000..e699dbd --- /dev/null +++ b/packages/api/.dockerignore @@ -0,0 +1,5 @@ +twister +*.exe +.env +.env.* +!.env.example diff --git a/packages/api/go.mod b/packages/api/go.mod index 335d36a..c34d6ee 100644 --- a/packages/api/go.mod +++ b/packages/api/go.mod @@ -3,6 +3,7 @@ module tangled.org/desertthunder.dev/twister go 1.25.0 require ( + github.com/charmbracelet/log v1.0.0 github.com/coder/websocket v1.8.12 github.com/joho/godotenv v1.5.1 github.com/spf13/cobra v1.10.2 @@ -12,13 +13,25 @@ require ( require ( github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc // indirect + github.com/charmbracelet/lipgloss v1.1.0 // indirect + github.com/charmbracelet/x/ansi v0.8.0 // indirect + github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logfmt/logfmt v0.6.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/muesli/termenv v0.16.0 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect golang.org/x/sys v0.42.0 // indirect modernc.org/libc v1.70.0 // indirect diff --git a/packages/api/go.sum b/packages/api/go.sum index 166b5e2..5096211 100644 --- a/packages/api/go.sum +++ b/packages/api/go.sum @@ -1,10 +1,30 @@ github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc h1:4pZI35227imm7yK2bGPcfpFEmuY1gc2YSTShr4iJBfs= +github.com/charmbracelet/colorprofile v0.2.3-0.20250311203215-f60798e515dc/go.mod h1:X4/0JoqgTIPSFcRA/P6INZzIuyqdFY5rm8tb41s9okk= +github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY= +github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30= +github.com/charmbracelet/log v1.0.0 h1:HVVVMmfOorfj3BA9i8X8UL69Hoz9lI0PYwXfJvOdRc4= +github.com/charmbracelet/log v1.0.0/go.mod h1:uYgY3SmLpwJWxmlrPwXvzVYujxis1vAKRV/0VQB7yWA= +github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= +github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd h1:vy0GVL4jeHEwG5YOXDmi86oYw2yuYUGqz6a8sLwg0X8= +github.com/charmbracelet/x/cellbuf v0.0.13-0.20250311204145-2c3ea96c31dd/go.mod h1:xe0nKWGd3eJgtqZRaN9RjMtK7xUYchjzPr7q6kcvCCs= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-logfmt/logfmt v0.6.1 h1:4hvbpePJKnIzH1B+8OR/JPbTx37NktoI9LE2QZBBkvE= +github.com/go-logfmt/logfmt v0.6.1/go.mod h1:EV2pOAQoZaT1ZXZbqDl5hrymndi4SY9ED9/z6CO0XAk= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -15,19 +35,34 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tursodatabase/libsql-client-go v0.0.0-20251219100830-236aa1ff8acc h1:lzi/5fg2EfinRlh3v//YyIhnc4tY7BTqazQGwb1ar+0= github.com/tursodatabase/libsql-client-go v0.0.0-20251219100830-236aa1ff8acc/go.mod h1:08inkKyguB6CGGssc/JzhmQWwBgFQBgjlYFjxjRh7nU= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= @@ -41,6 +76,8 @@ golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v4 v4.32.0 h1:hjG66bI/kqIPX1b2yT6fr/jt+QedtP2fqojG2VrFuVw= diff --git a/packages/api/internal/config/config.go b/packages/api/internal/config/config.go index cb9155c..3195fa4 100644 --- a/packages/api/internal/config/config.go +++ b/packages/api/internal/config/config.go @@ -28,6 +28,7 @@ type Config struct { HybridKeywordWeight float64 HybridSemanticWeight float64 HTTPBindAddr string + IndexerHealthAddr string LogLevel string LogFormat string EnableAdminEndpoints bool @@ -49,6 +50,7 @@ func Load() (*Config, error) { EmbeddingAPIKey: os.Getenv("EMBEDDING_API_KEY"), EmbeddingAPIURL: os.Getenv("EMBEDDING_API_URL"), HTTPBindAddr: envOrDefault("HTTP_BIND_ADDR", ":8080"), + IndexerHealthAddr: envOrDefault("INDEXER_HEALTH_ADDR", ":9090"), LogLevel: envOrDefault("LOG_LEVEL", "info"), LogFormat: envOrDefault("LOG_FORMAT", "json"), AdminAuthToken: os.Getenv("ADMIN_AUTH_TOKEN"), diff --git a/packages/api/internal/ingest/ingest_test.go b/packages/api/internal/ingest/ingest_test.go index dff7c60..6f3c49e 100644 --- a/packages/api/internal/ingest/ingest_test.go +++ b/packages/api/internal/ingest/ingest_test.go @@ -115,6 +115,10 @@ func (f *fakeStore) CountDocuments(_ context.Context) (int64, error) { return int64(len(f.docs)), nil } +func (f *fakeStore) Ping(_ context.Context) error { + return nil +} + func newRunnerForTest(st *fakeStore, tap *fakeTapClient, indexedCollections string) *Runner { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) return NewRunner(st, normalize.NewRegistry(), tap, indexedCollections, logger) diff --git a/packages/api/internal/observability/log.go b/packages/api/internal/observability/log.go index 8227772..f54b6c8 100644 --- a/packages/api/internal/observability/log.go +++ b/packages/api/internal/observability/log.go @@ -4,27 +4,29 @@ import ( "log/slog" "os" + charmlog "github.com/charmbracelet/log" "tangled.org/desertthunder.dev/twister/internal/config" ) func NewLogger(cfg *config.Config) *slog.Logger { - level := slog.LevelInfo + level := charmlog.InfoLevel switch cfg.LogLevel { case "debug": - level = slog.LevelDebug + level = charmlog.DebugLevel case "warn": - level = slog.LevelWarn + level = charmlog.WarnLevel case "error": - level = slog.LevelError + level = charmlog.ErrorLevel } - opts := &slog.HandlerOptions{Level: level} - var handler slog.Handler - if cfg.LogFormat == "text" { - handler = slog.NewTextHandler(os.Stdout, opts) + if cfg.LogFormat == "json" { + handler = slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.Level(level)}) } else { - handler = slog.NewJSONHandler(os.Stdout, opts) + handler = charmlog.NewWithOptions(os.Stdout, charmlog.Options{ + Level: level, + ReportTimestamp: true, + }) } return slog.New(handler) diff --git a/packages/api/internal/store/sql_store.go b/packages/api/internal/store/sql_store.go index b686c3a..332b126 100644 --- a/packages/api/internal/store/sql_store.go +++ b/packages/api/internal/store/sql_store.go @@ -251,6 +251,10 @@ func (s *SQLStore) CountDocuments(ctx context.Context) (int64, error) { return n, nil } +func (s *SQLStore) Ping(ctx context.Context) error { + return s.db.PingContext(ctx) +} + func scanDocument(row *sql.Row) (*Document, error) { doc := &Document{} var ( diff --git a/packages/api/internal/store/store.go b/packages/api/internal/store/store.go index a1a5efc..d5a60f7 100644 --- a/packages/api/internal/store/store.go +++ b/packages/api/internal/store/store.go @@ -54,4 +54,5 @@ type Store interface { GetFollowSubjects(ctx context.Context, did string) ([]string, error) GetRepoCollaborators(ctx context.Context, repoOwnerDID string) ([]string, error) CountDocuments(ctx context.Context) (int64, error) + Ping(ctx context.Context) error } diff --git a/packages/api/main.go b/packages/api/main.go index 0c42122..0ba2eeb 100644 --- a/packages/api/main.go +++ b/packages/api/main.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "net/http" "os/signal" "syscall" "time" @@ -136,6 +137,35 @@ func newIndexerCmd() *cobra.Command { ctx, cancel := baseContext() defer cancel() + // Start health server on separate port for Railway health checks. + healthMux := http.NewServeMux() + healthMux.HandleFunc("GET /health", func(w http.ResponseWriter, r *http.Request) { + if err := st.Ping(r.Context()); err != nil { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + fmt.Fprintf(w, `{"status":"unhealthy","error":"db_unreachable"}`) + return + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"status":"ok"}`) + }) + healthSrv := &http.Server{ + Addr: cfg.IndexerHealthAddr, + Handler: healthMux, + ReadHeaderTimeout: 5 * time.Second, + } + go func() { + log.Info("indexer health server listening", slog.String("addr", cfg.IndexerHealthAddr)) + if err := healthSrv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("indexer health server failed", slog.String("error", err.Error())) + } + }() + defer func() { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer shutdownCancel() + _ = healthSrv.Shutdown(shutdownCtx) + }() + if err := runner.Run(ctx); err != nil { return fmt.Errorf("run indexer: %w", err) } -- 2.51.2