diff --git a/README.md b/README.md index 03e291c..76e5da0 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,62 @@ just api-run-api To enable indexed search in the client, set `VITE_TWISTER_API_BASE_URL` in `apps/twisted/.env`. +## Run Locally + +Install dependencies once from the repo root: + +```bash +pnpm install +``` + +Start the Ionic/Vite app: + +```bash +pnpm dev +# or: just dev +``` + +That serves the client from `apps/twisted` with Vite. + +To run the Go API locally, make sure `packages/api/.env` has at least: + +- `TURSO_DATABASE_URL` +- `TURSO_AUTH_TOKEN` + +Then start the API: + +```bash +pnpm api:run:api +# or: just api-dev +``` + +This serves the API and search site on `http://localhost:8080`. + +To run the indexer as well, `packages/api/.env` also needs: + +- `TAP_URL` +- `TAP_AUTH_PASSWORD` +- `INDEXED_COLLECTIONS` + +Then start the indexer in a separate terminal: + +```bash +pnpm api:run:indexer +# or: just api-run-indexer +``` + +Typical local setup is three terminals: + +1. `pnpm dev` +2. `pnpm api:run:api` +3. `pnpm api:run:indexer` + +If you want the app to call the local API, set this in `apps/twisted/.env`: + +```bash +VITE_TWISTER_API_BASE_URL=http://localhost:8080 +``` + ## Infrastructure Setup ### Turso diff --git a/packages/api/internal/normalize/normalize.go b/packages/api/internal/normalize/normalize.go index 06e991b..632a062 100644 --- a/packages/api/internal/normalize/normalize.go +++ b/packages/api/internal/normalize/normalize.go @@ -108,6 +108,15 @@ func str(m map[string]any, key string) string { return "" } +func firstString(m map[string]any, keys ...string) string { + for _, key := range keys { + if v := str(m, key); v != "" { + return v + } + } + return "" +} + // nestedMap safely extracts a nested map[string]any from a map. func nestedMap(m map[string]any, key string) map[string]any { if v, ok := m[key]; ok { diff --git a/packages/api/internal/normalize/normalize_test.go b/packages/api/internal/normalize/normalize_test.go index ef91f74..b20ae5e 100644 --- a/packages/api/internal/normalize/normalize_test.go +++ b/packages/api/internal/normalize/normalize_test.go @@ -394,7 +394,7 @@ func TestIssueStateHandler_MissingFields(t *testing.T) { t.Run("missing status", func(t *testing.T) { event := normalize.TapRecordEvent{ Record: &normalize.TapRecord{ - Record: map[string]any{"subject": "at://did:plc:x/col/rkey"}, + Record: map[string]any{"issue": "at://did:plc:x/col/rkey"}, }, } _, err := handler.HandleState(event) @@ -402,6 +402,24 @@ func TestIssueStateHandler_MissingFields(t *testing.T) { t.Error("expected error for missing status") } }) + + t.Run("legacy field names still work", func(t *testing.T) { + event := normalize.TapRecordEvent{ + Record: &normalize.TapRecord{ + Record: map[string]any{ + "subject": "at://did:plc:x/col/rkey", + "status": "closed", + }, + }, + } + update, err := handler.HandleState(event) + if err != nil { + t.Fatalf("HandleState legacy: %v", err) + } + if update.SubjectURI != "at://did:plc:x/col/rkey" || update.State != "closed" { + t.Fatalf("legacy update = %#v", update) + } + }) } // TestRegistry verifies adapter and state handler lookup. diff --git a/packages/api/internal/normalize/state.go b/packages/api/internal/normalize/state.go index da43d4f..019add0 100644 --- a/packages/api/internal/normalize/state.go +++ b/packages/api/internal/normalize/state.go @@ -16,13 +16,13 @@ func (h *IssueStateHandler) HandleState(event TapRecordEvent) (*StateUpdate, err r := event.Record rec := r.Record - subject := str(rec, "subject") + subject := firstString(rec, "issue", "subject") if subject == "" { - return nil, fmt.Errorf("issue state record missing subject field") + return nil, fmt.Errorf("issue state record missing issue field") } - state := str(rec, "status") + state := firstString(rec, "state", "status") if state == "" { - return nil, fmt.Errorf("issue state record missing status field") + return nil, fmt.Errorf("issue state record missing state field") } return &StateUpdate{ @@ -40,11 +40,11 @@ func (h *PullStatusHandler) HandleState(event TapRecordEvent) (*StateUpdate, err r := event.Record rec := r.Record - subject := str(rec, "subject") + subject := firstString(rec, "pull", "subject") if subject == "" { - return nil, fmt.Errorf("pull status record missing subject field") + return nil, fmt.Errorf("pull status record missing pull field") } - status := str(rec, "status") + status := firstString(rec, "status", "state") if status == "" { return nil, fmt.Errorf("pull status record missing status field") } diff --git a/packages/api/internal/normalize/testdata/issue_state.json b/packages/api/internal/normalize/testdata/issue_state.json index 7cc26ca..798362a 100644 --- a/packages/api/internal/normalize/testdata/issue_state.json +++ b/packages/api/internal/normalize/testdata/issue_state.json @@ -11,8 +11,8 @@ "cid": "bafyreigabc128", "record": { "$type": "sh.tangled.repo.issue.state", - "subject": "at://did:plc:abc123/sh.tangled.repo.issue/3kb3fge5lm32y", - "status": "closed" + "issue": "at://did:plc:abc123/sh.tangled.repo.issue/3kb3fge5lm32y", + "state": "closed" } } } diff --git a/packages/api/internal/normalize/testdata/pull_status.json b/packages/api/internal/normalize/testdata/pull_status.json index e3028ba..6195a71 100644 --- a/packages/api/internal/normalize/testdata/pull_status.json +++ b/packages/api/internal/normalize/testdata/pull_status.json @@ -11,7 +11,7 @@ "cid": "bafyreigabc129", "record": { "$type": "sh.tangled.repo.pull.status", - "subject": "at://did:plc:abc123/sh.tangled.repo.pull/3kb3fge5lm32z", + "pull": "at://did:plc:abc123/sh.tangled.repo.pull/3kb3fge5lm32z", "status": "merged" } } diff --git a/packages/api/internal/search/search.go b/packages/api/internal/search/search.go index 4fce8a7..bf097dd 100644 --- a/packages/api/internal/search/search.go +++ b/packages/api/internal/search/search.go @@ -31,6 +31,7 @@ type Result struct { BodySnippet string `json:"body_snippet,omitempty"` Summary string `json:"summary,omitempty"` RepoName string `json:"repo_name,omitempty"` + RepoOwnerHandle string `json:"repo_owner_handle,omitempty"` AuthorHandle string `json:"author_handle,omitempty"` DID string `json:"did"` ATURI string `json:"at_uri"` @@ -126,12 +127,13 @@ func (r *Repository) Keyword(ctx context.Context, p Params) (*Response, error) { // Fetch results with score and snippet. resultsSQL := fmt.Sprintf(` - SELECT d.id, d.title, d.summary, d.repo_name, d.author_handle, + SELECT d.id, d.title, d.summary, d.repo_name, repo_owner.handle, d.author_handle, d.did, d.at_uri, d.collection, d.record_type, d.created_at, d.updated_at, -bm25(documents_fts, 0.0, 3.0, 1.0, 1.5, 2.5, 2.0, 1.2) AS score, snippet(documents_fts, 2, '', '', '...', 20) AS body_snippet FROM documents_fts JOIN documents d ON d.id = documents_fts.id + LEFT JOIN identity_handles repo_owner ON repo_owner.did = d.repo_did AND repo_owner.is_active = 1 %s WHERE %s ORDER BY score DESC @@ -151,12 +153,12 @@ func (r *Repository) Keyword(ctx context.Context, p Params) (*Response, error) { results := make([]Result, 0) for rows.Next() { var res Result - var title, summary, repoName, authorHandle sql.NullString + var title, summary, repoName, repoOwnerHandle, authorHandle sql.NullString var createdAt, updatedAt sql.NullString var bodySnippet sql.NullString if err := rows.Scan( - &res.ID, &title, &summary, &repoName, &authorHandle, + &res.ID, &title, &summary, &repoName, &repoOwnerHandle, &authorHandle, &res.DID, &res.ATURI, &res.Collection, &res.RecordType, &createdAt, &updatedAt, &res.Score, &bodySnippet, ); err != nil { @@ -165,6 +167,7 @@ func (r *Repository) Keyword(ctx context.Context, p Params) (*Response, error) { res.Title = title.String res.Summary = summary.String res.RepoName = repoName.String + res.RepoOwnerHandle = repoOwnerHandle.String res.AuthorHandle = authorHandle.String res.BodySnippet = bodySnippet.String res.CreatedAt = createdAt.String diff --git a/packages/api/internal/tapclient/tapclient.go b/packages/api/internal/tapclient/tapclient.go index 8b20840..e0badfe 100644 --- a/packages/api/internal/tapclient/tapclient.go +++ b/packages/api/internal/tapclient/tapclient.go @@ -21,8 +21,9 @@ import ( const ( minReconnectBackoff = 500 * time.Millisecond maxReconnectBackoff = 10 * time.Second - keepAliveInterval = 20 * time.Second - keepAliveTimeout = 5 * time.Second + keepAliveInterval = 2 * time.Minute + keepAliveTimeout = 20 * time.Second + maxReadMessageBytes = 8 << 20 ) // Client receives Tap events over WebSocket and sends acks after processing. @@ -31,10 +32,11 @@ type Client struct { password string log *slog.Logger - mu sync.Mutex - conn *websocket.Conn - ackAsJSON bool - disableAcks bool + mu sync.Mutex + conn *websocket.Conn + ackAsJSON bool + disableAcks bool + lastActivity time.Time } func New(url, password string, log *slog.Logger) *Client { @@ -43,11 +45,12 @@ func New(url, password string, log *slog.Logger) *Client { } disableAcks, _ := strconv.ParseBool(strings.TrimSpace(os.Getenv("TAP_DISABLE_ACKS"))) return &Client{ - url: url, - password: password, - log: log, - ackAsJSON: true, - disableAcks: disableAcks, + url: url, + password: password, + log: log, + ackAsJSON: true, + disableAcks: disableAcks, + lastActivity: time.Now(), } } @@ -68,6 +71,8 @@ func (c *Client) ReadEvent(ctx context.Context) (normalize.TapRecordEvent, error continue } + c.markActivity() + var event normalize.TapRecordEvent if err := json.Unmarshal(data, &event); err != nil { c.log.Warn("tap decode failed", slog.String("error", err.Error())) @@ -95,6 +100,7 @@ func (c *Client) AckEvent(ctx context.Context, id int64) error { if ackAsJSON { payload, _ := json.Marshal(map[string]int64{"id": id}) if err := conn.Write(ctx, websocket.MessageText, payload); err == nil { + c.markActivity() return nil } else if isConnectionWriteError(err) { c.resetConn(websocket.StatusInternalError, "ack json write failed") @@ -109,6 +115,7 @@ func (c *Client) AckEvent(ctx context.Context, id int64) error { c.resetConn(websocket.StatusInternalError, "ack failed") return fmt.Errorf("ack event %d: %w", id, err) } + c.markActivity() c.mu.Lock() c.ackAsJSON = false @@ -120,6 +127,7 @@ func (c *Client) AckEvent(ctx context.Context, id int64) error { c.resetConn(websocket.StatusInternalError, "ack failed") return fmt.Errorf("ack event %d: %w", id, err) } + c.markActivity() return nil } @@ -157,9 +165,11 @@ func (c *Client) ensureConnected(ctx context.Context) (*websocket.Conn, error) { conn, _, err := websocket.Dial(ctx, c.url, &websocket.DialOptions{HTTPHeader: h}) if err == nil { + conn.SetReadLimit(maxReadMessageBytes) c.mu.Lock() if c.conn == nil { c.conn = conn + c.lastActivity = time.Now() c.startKeepAlive(conn) } else { _ = conn.Close(websocket.StatusNormalClosure, "duplicate") @@ -187,6 +197,12 @@ func (c *Client) ensureConnected(ctx context.Context) (*websocket.Conn, error) { } } +func (c *Client) markActivity() { + c.mu.Lock() + c.lastActivity = time.Now() + c.mu.Unlock() +} + func (c *Client) resetConn(status websocket.StatusCode, reason string) { c.mu.Lock() defer c.mu.Unlock() @@ -208,6 +224,10 @@ func (c *Client) startKeepAlive(conn *websocket.Conn) { c.mu.Unlock() return } + if time.Since(c.lastActivity) < keepAliveInterval { + c.mu.Unlock() + continue + } c.mu.Unlock() ctx, cancel := context.WithTimeout(context.Background(), keepAliveTimeout) diff --git a/packages/api/internal/view/static/search.js b/packages/api/internal/view/static/search.js index 2effd29..ce251c1 100644 --- a/packages/api/internal/view/static/search.js +++ b/packages/api/internal/view/static/search.js @@ -1,4 +1,6 @@ function searchApp() { + const TANGLED_BASE = "https://tangled.org"; + return { query: "", filters: { type: "", author: "", language: "", state: "" }, @@ -85,21 +87,53 @@ function searchApp() { }, canonicalURL(r) { - const h = r.author_handle || ""; + const explicitURL = this.extractTangledURL(r.body_snippet) || this.extractTangledURL(r.summary); + if (explicitURL) return explicitURL; + + const author = this.normalizeOwner(r.author_handle); + const repoOwner = this.normalizeOwner(r.repo_owner_handle) || author; + const repoName = this.normalizeSegment(r.repo_name); + switch (r.record_type) { + case "profile": + return author ? this.buildTangledURL(author) : "#"; case "repo": - return h && r.repo_name ? "https://tangled.org/" + h + "/" + r.repo_name : "#"; + return repoOwner && repoName ? this.buildTangledURL(repoOwner, repoName) : "#"; case "issue": - return h && r.repo_name ? "https://tangled.org/" + h + "/" + r.repo_name + "/issues" : "#"; + case "issue_comment": + return repoOwner && repoName ? this.buildTangledURL(repoOwner, repoName, "issues") : "#"; case "pull": - return h && r.repo_name ? "https://tangled.org/" + h + "/" + r.repo_name + "/pulls" : "#"; - case "profile": - return h ? "https://tangled.org/" + h : "#"; + case "pull_comment": + return repoOwner && repoName ? this.buildTangledURL(repoOwner, repoName, "pulls") : "#"; + case "string": + return author ? this.buildTangledURL(author) : "#"; default: return "#"; } }, + buildTangledURL() { + const segments = Array.from(arguments) + .filter(Boolean) + .map((segment) => encodeURIComponent(segment)); + return TANGLED_BASE + "/" + segments.join("/"); + }, + + normalizeOwner(owner) { + return owner ? owner.replace(/^@+/, "").trim() : ""; + }, + + normalizeSegment(segment) { + return segment ? segment.trim() : ""; + }, + + extractTangledURL(text) { + if (!text) return ""; + const match = text.match(/https:\/\/tangled\.org\/[^\s<>"']+/i); + if (!match) return ""; + return match[0].replace(/[),.;:>]+$/, ""); + }, + relTime(iso) { if (!iso) return ""; const diff = Date.now() - new Date(iso).getTime(); diff --git a/packages/api/internal/view/static/style.css b/packages/api/internal/view/static/style.css index 4d767c0..998b245 100644 --- a/packages/api/internal/view/static/style.css +++ b/packages/api/internal/view/static/style.css @@ -54,6 +54,7 @@ a:hover { text-decoration: underline; } margin: 0 auto; padding: 2rem 1rem; flex: 1; + min-width: 0; } /* Footer */ @@ -132,9 +133,17 @@ a:hover { text-decoration: underline; } margin-bottom: .6rem; color: var(--text); transition: border-color .15s; + min-width: 0; } .card:hover { border-color: var(--accent); text-decoration: none; } -.card-head { display: flex; align-items: center; gap: .5rem; margin-bottom: .35rem; } +.card-head { + display: flex; + align-items: flex-start; + gap: .5rem; + margin-bottom: .35rem; + flex-wrap: wrap; + min-width: 0; +} .badge { font-family: var(--mono); font-size: .7rem; @@ -145,12 +154,20 @@ a:hover { text-decoration: underline; } color: var(--text-dim); white-space: nowrap; } -.card-title { font-weight: 500; font-size: .95rem; } +.card-title { + font-weight: 500; + font-size: .95rem; + min-width: 0; + overflow-wrap: anywhere; + word-break: break-word; +} .card-snippet { font-size: .85rem; color: var(--text-dim); margin-bottom: .35rem; line-height: 1.5; + overflow-wrap: anywhere; + word-break: break-word; } .card-snippet mark { background: var(--mark-bg); @@ -158,7 +175,19 @@ a:hover { text-decoration: underline; } padding: 0 .1rem; border-radius: 2px; } -.card-meta { font-size: .78rem; color: var(--text-dim); display: flex; gap: .5rem; } +.card-meta { + font-size: .78rem; + color: var(--text-dim); + display: flex; + gap: .5rem; + flex-wrap: wrap; + min-width: 0; +} +.card-meta span { + min-width: 0; + overflow-wrap: anywhere; + word-break: break-word; +} .meta-sep::before { content: "\00b7"; margin-right: .5rem; } /* Docs */ @@ -171,6 +200,8 @@ a:hover { text-decoration: underline; } border-collapse: collapse; margin-bottom: 1rem; font-size: .85rem; + display: block; + overflow-x: auto; } .main th, .main td { text-align: left; @@ -196,10 +227,19 @@ a:hover { text-decoration: underline; } line-height: 1.5; } .main pre code { background: none; padding: 0; } +.main p, +.main li, +.main td, +.main th, +.main code { + overflow-wrap: anywhere; + word-break: break-word; +} /* Mobile */ @media (max-width: 640px) { .search-form { flex-direction: column; } + .btn-primary { width: 100%; } .filter-bar { flex-direction: column; } .filter-input { width: 100%; } .card-meta { flex-wrap: wrap; } diff --git a/packages/api/internal/view/view.go b/packages/api/internal/view/view.go index b26414f..1c27b7e 100644 --- a/packages/api/internal/view/view.go +++ b/packages/api/internal/view/view.go @@ -10,17 +10,22 @@ import ( //go:embed templates static var content embed.FS -var templates *template.Template +var pageTemplates map[string]*template.Template func init() { - templates = template.Must(template.ParseFS(content, - "templates/layout.html", - "templates/index.html", - "templates/docs/index.html", - "templates/docs/search.html", - "templates/docs/documents.html", - "templates/docs/health.html", - )) + pageTemplates = make(map[string]*template.Template) + for _, name := range []string{ + "index.html", + "docs/index.html", + "docs/search.html", + "docs/documents.html", + "docs/health.html", + } { + pageTemplates[name] = template.Must(template.New("layout").ParseFS(content, + "templates/layout.html", + "templates/"+name, + )) + } } // Handler returns an http.Handler that serves the site pages and static assets. @@ -41,8 +46,14 @@ func Handler() http.Handler { func renderPage(name string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + tmpl, ok := pageTemplates[name] + if !ok { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") - if err := templates.ExecuteTemplate(w, name, nil); err != nil { + if err := tmpl.ExecuteTemplate(w, "layout", nil); err != nil { http.Error(w, "template error", http.StatusInternalServerError) } } diff --git a/packages/api/internal/view/view_test.go b/packages/api/internal/view/view_test.go new file mode 100644 index 0000000..6977c0a --- /dev/null +++ b/packages/api/internal/view/view_test.go @@ -0,0 +1,46 @@ +package view + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func TestHandlerRendersSearchHome(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + + Handler().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", rec.Code) + } + + body := rec.Body.String() + if !strings.Contains(body, "Search Tangled") { + t.Fatalf("expected search home content, got body %q", body) + } + if strings.Contains(body, "Health Endpoints") { + t.Fatalf("expected search home page, got health page content") + } +} + +func TestHandlerRendersDocsIndex(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/docs", nil) + rec := httptest.NewRecorder() + + Handler().ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("expected status 200, got %d", rec.Code) + } + + body := rec.Body.String() + if !strings.Contains(body, "API Documentation") { + t.Fatalf("expected docs page content, got body %q", body) + } + if strings.Contains(body, "template error") { + t.Fatalf("expected docs page render, got template error") + } +}