diff --git a/internal/core/unfurl/circuit_breaker.go b/internal/core/unfurl/circuit_breaker.go new file mode 100644 index 0000000..54815f9 --- /dev/null +++ b/internal/core/unfurl/circuit_breaker.go @@ -0,0 +1,200 @@ +package unfurl + +import ( + "fmt" + "log" + "sync" + "time" +) + +// circuitState represents the state of a circuit breaker +type circuitState int + +const ( + stateClosed circuitState = iota // Normal operation + stateOpen // Circuit is open (provider failing) + stateHalfOpen // Testing if provider recovered +) + +// circuitBreaker tracks failures per provider and stops trying failing providers +type circuitBreaker struct { + failures map[string]int + lastFailure map[string]time.Time + state map[string]circuitState + lastStateLog map[string]time.Time + failureThreshold int + openDuration time.Duration + mu sync.RWMutex +} + +// newCircuitBreaker creates a circuit breaker with default settings +func newCircuitBreaker() *circuitBreaker { + return &circuitBreaker{ + failureThreshold: 3, // Open after 3 consecutive failures + openDuration: 5 * time.Minute, // Keep open for 5 minutes + failures: make(map[string]int), + lastFailure: make(map[string]time.Time), + state: make(map[string]circuitState), + lastStateLog: make(map[string]time.Time), + } +} + +// canAttempt checks if we should attempt to call this provider +// Returns true if circuit is closed or half-open (ready to retry) +func (cb *circuitBreaker) canAttempt(provider string) (bool, error) { + cb.mu.RLock() + defer cb.mu.RUnlock() + + state := cb.getState(provider) + + switch state { + case stateClosed: + return true, nil + case stateOpen: + // Check if we should transition to half-open + lastFail := cb.lastFailure[provider] + if time.Since(lastFail) > cb.openDuration { + // Transition to half-open (allow one retry) + cb.mu.RUnlock() + cb.mu.Lock() + cb.state[provider] = stateHalfOpen + cb.logStateChange(provider, stateHalfOpen) + cb.mu.Unlock() + cb.mu.RLock() + return true, nil + } + // Still in open period + failCount := cb.failures[provider] + nextRetry := lastFail.Add(cb.openDuration) + return false, fmt.Errorf( + "circuit breaker open for provider '%s' (failures: %d, next retry: %s)", + provider, + failCount, + nextRetry.Format("15:04:05"), + ) + case stateHalfOpen: + return true, nil + default: + return true, nil + } +} + +// recordSuccess records a successful unfurl, resetting failure count +func (cb *circuitBreaker) recordSuccess(provider string) { + cb.mu.Lock() + defer cb.mu.Unlock() + + oldState := cb.getState(provider) + + // Reset failure tracking + delete(cb.failures, provider) + delete(cb.lastFailure, provider) + cb.state[provider] = stateClosed + + // Log recovery if we were in a failure state + if oldState != stateClosed { + cb.logStateChange(provider, stateClosed) + } +} + +// recordFailure records a failed unfurl attempt +func (cb *circuitBreaker) recordFailure(provider string, err error) { + cb.mu.Lock() + defer cb.mu.Unlock() + + // Increment failure count + cb.failures[provider]++ + cb.lastFailure[provider] = time.Now() + + failCount := cb.failures[provider] + + // Check if we should open the circuit + if failCount >= cb.failureThreshold { + oldState := cb.getState(provider) + cb.state[provider] = stateOpen + if oldState != stateOpen { + log.Printf( + "[UNFURL-CIRCUIT] Opening circuit for provider '%s' after %d consecutive failures. Last error: %v", + provider, + failCount, + err, + ) + cb.lastStateLog[provider] = time.Now() + } + } else { + log.Printf( + "[UNFURL-CIRCUIT] Failure %d/%d for provider '%s': %v", + failCount, + cb.failureThreshold, + provider, + err, + ) + } +} + +// getState returns the current state (must be called with lock held) +func (cb *circuitBreaker) getState(provider string) circuitState { + if state, exists := cb.state[provider]; exists { + return state + } + return stateClosed +} + +// logStateChange logs state transitions (must be called with lock held) +// Debounced to avoid log spam (max once per minute per provider) +func (cb *circuitBreaker) logStateChange(provider string, newState circuitState) { + lastLog, exists := cb.lastStateLog[provider] + if exists && time.Since(lastLog) < time.Minute { + return // Don't spam logs + } + + var stateStr string + switch newState { + case stateClosed: + stateStr = "CLOSED (recovered)" + case stateOpen: + stateStr = "OPEN (failing)" + case stateHalfOpen: + stateStr = "HALF-OPEN (testing)" + } + + log.Printf("[UNFURL-CIRCUIT] Circuit for provider '%s' is now %s", provider, stateStr) + cb.lastStateLog[provider] = time.Now() +} + +// getStats returns current circuit breaker stats (for debugging/monitoring) +func (cb *circuitBreaker) getStats() map[string]interface{} { + cb.mu.RLock() + defer cb.mu.RUnlock() + + stats := make(map[string]interface{}) + + // Collect all providers with any activity (state, failures, or both) + providers := make(map[string]bool) + for provider := range cb.state { + providers[provider] = true + } + for provider := range cb.failures { + providers[provider] = true + } + + for provider := range providers { + state := cb.getState(provider) + var stateStr string + switch state { + case stateClosed: + stateStr = "closed" + case stateOpen: + stateStr = "open" + case stateHalfOpen: + stateStr = "half-open" + } + + stats[provider] = map[string]interface{}{ + "state": stateStr, + "failures": cb.failures[provider], + "last_failure": cb.lastFailure[provider], + } + } + return stats +} diff --git a/internal/core/unfurl/circuit_breaker_test.go b/internal/core/unfurl/circuit_breaker_test.go new file mode 100644 index 0000000..34cf01b --- /dev/null +++ b/internal/core/unfurl/circuit_breaker_test.go @@ -0,0 +1,175 @@ +package unfurl + +import ( + "fmt" + "testing" + "time" +) + +func TestCircuitBreaker_Basic(t *testing.T) { + cb := newCircuitBreaker() + + provider := "test-provider" + + // Should start closed (allow attempts) + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to be closed initially, but got error: %v", err) + } + + // Record success + cb.recordSuccess(provider) + canAttempt, _ = cb.canAttempt(provider) + if !canAttempt { + t.Error("Expected circuit to remain closed after success") + } +} + +func TestCircuitBreaker_OpensAfterFailures(t *testing.T) { + cb := newCircuitBreaker() + provider := "failing-provider" + + // Record failures up to threshold + for i := 0; i < cb.failureThreshold; i++ { + cb.recordFailure(provider, fmt.Errorf("test error %d", i)) + } + + // Circuit should now be open + canAttempt, err := cb.canAttempt(provider) + if canAttempt { + t.Error("Expected circuit to be open after threshold failures") + } + if err == nil { + t.Error("Expected error when circuit is open") + } +} + +func TestCircuitBreaker_RecoveryAfterSuccess(t *testing.T) { + cb := newCircuitBreaker() + provider := "recovery-provider" + + // Record some failures + cb.recordFailure(provider, fmt.Errorf("error 1")) + cb.recordFailure(provider, fmt.Errorf("error 2")) + + // Record success - should reset failure count + cb.recordSuccess(provider) + + // Should be able to attempt again + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to be closed after success, but got error: %v", err) + } + + // Failure count should be reset + if count := cb.failures[provider]; count != 0 { + t.Errorf("Expected failure count to be reset to 0, got %d", count) + } +} + +func TestCircuitBreaker_HalfOpenTransition(t *testing.T) { + cb := newCircuitBreaker() + cb.openDuration = 100 * time.Millisecond // Short duration for testing + provider := "half-open-provider" + + // Open the circuit + for i := 0; i < cb.failureThreshold; i++ { + cb.recordFailure(provider, fmt.Errorf("error %d", i)) + } + + // Should be open + canAttempt, _ := cb.canAttempt(provider) + if canAttempt { + t.Error("Expected circuit to be open") + } + + // Wait for open duration + time.Sleep(150 * time.Millisecond) + + // Should transition to half-open and allow one attempt + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to transition to half-open after duration, but got error: %v", err) + } + + // State should be half-open + cb.mu.RLock() + state := cb.state[provider] + cb.mu.RUnlock() + + if state != stateHalfOpen { + t.Errorf("Expected state to be half-open, got %v", state) + } +} + +func TestCircuitBreaker_MultipleProviders(t *testing.T) { + cb := newCircuitBreaker() + + // Open circuit for provider A + for i := 0; i < cb.failureThreshold; i++ { + cb.recordFailure("providerA", fmt.Errorf("error")) + } + + // Provider A should be blocked + canAttemptA, _ := cb.canAttempt("providerA") + if canAttemptA { + t.Error("Expected providerA circuit to be open") + } + + // Provider B should still be open (independent circuits) + canAttemptB, err := cb.canAttempt("providerB") + if !canAttemptB { + t.Errorf("Expected providerB circuit to be closed, but got error: %v", err) + } +} + +func TestCircuitBreaker_GetStats(t *testing.T) { + cb := newCircuitBreaker() + + // Record some activity + cb.recordFailure("provider1", fmt.Errorf("error 1")) + cb.recordFailure("provider1", fmt.Errorf("error 2")) + + stats := cb.getStats() + + // Should have stats for providers with failures + if providerStats, ok := stats["provider1"]; !ok { + t.Error("Expected stats for provider1") + } else { + // Check that failure count is tracked + statsMap := providerStats.(map[string]interface{}) + if failures, ok := statsMap["failures"].(int); !ok || failures != 2 { + t.Errorf("Expected 2 failures for provider1, got %v", statsMap["failures"]) + } + } + + // Provider that succeeds is cleaned up from state + cb.recordSuccess("provider2") + _ = cb.getStats() + // Provider2 should not be in stats (or have state "closed" with 0 failures) +} + +func TestCircuitBreaker_FailureThresholdExact(t *testing.T) { + cb := newCircuitBreaker() + provider := "exact-threshold-provider" + + // Record failures just below threshold + for i := 0; i < cb.failureThreshold-1; i++ { + cb.recordFailure(provider, fmt.Errorf("error %d", i)) + } + + // Should still be closed + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to be closed below threshold, but got error: %v", err) + } + + // One more failure should open it + cb.recordFailure(provider, fmt.Errorf("final error")) + + // Should now be open + canAttempt, _ = cb.canAttempt(provider) + if canAttempt { + t.Error("Expected circuit to be open at threshold") + } +} diff --git a/internal/core/unfurl/kagi_test.go b/internal/core/unfurl/kagi_test.go new file mode 100644 index 0000000..f5e79a1 --- /dev/null +++ b/internal/core/unfurl/kagi_test.go @@ -0,0 +1,202 @@ +package unfurl + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchKagiKite_Success(t *testing.T) { + // Mock Kagi HTML response + mockHTML := ` + +
+No images here
+` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "no image found") +} + +func TestFetchKagiKite_FallbackToTitle(t *testing.T) { + mockHTML := ` + +
+
+`
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/html")
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write([]byte(mockHTML))
+ }))
+ defer server.Close()
+
+ ctx := context.Background()
+
+ result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0")
+
+ assert.Error(t, err)
+ assert.Nil(t, result)
+ assert.Contains(t, err.Error(), "no image found")
+}
diff --git a/internal/core/unfurl/opengraph_test.go b/internal/core/unfurl/opengraph_test.go
new file mode 100644
index 0000000..cb490f6
--- /dev/null
+++ b/internal/core/unfurl/opengraph_test.go
@@ -0,0 +1,269 @@
+package unfurl
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestParseOpenGraph_ValidTags(t *testing.T) {
+ html := `
+
+
+
+
+
+
+
+
+
+ Some content
+ + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Equal(t, "Test Article Title", og.Title) + assert.Equal(t, "This is a test description", og.Description) + assert.Equal(t, "https://example.com/image.jpg", og.Image) + assert.Equal(t, "https://example.com/canonical", og.URL) +} + +func TestParseOpenGraph_MissingImage(t *testing.T) { + html := ` + + + + + + + + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Equal(t, "Article Without Image", og.Title) + assert.Equal(t, "No image tag", og.Description) + assert.Empty(t, og.Image, "Image should be empty when not provided") +} + +func TestParseOpenGraph_FallbackToTitle(t *testing.T) { + html := ` + + + +Article content
+ +` + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(html)) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + require.NoError(t, err) + require.NotNil(t, result) + + assert.Equal(t, "Test News Article", result.Title) + assert.Equal(t, "Breaking news story", result.Description) + assert.Equal(t, "https://example.com/news.jpg", result.ThumbnailURL) + assert.Equal(t, "article", result.Type) + assert.Equal(t, "opengraph", result.Provider) +} + +func TestFetchOpenGraph_HTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "404") +} + +func TestFetchOpenGraph_Timeout(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 100*time.Millisecond, "CovesBot/1.0") + require.Error(t, err) + assert.Nil(t, result) +} + +func TestFetchOpenGraph_NoMetadata(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + html := `No metadata
` + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(html)) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + require.NoError(t, err) + require.NotNil(t, result) + + // Should still return a result with domain + assert.Equal(t, "article", result.Type) + assert.Equal(t, "opengraph", result.Provider) + assert.NotEmpty(t, result.Domain) +} + +func TestIsOEmbedProvider(t *testing.T) { + tests := []struct { + url string + expected bool + }{ + {"https://streamable.com/abc123", true}, + {"https://www.youtube.com/watch?v=test", true}, + {"https://youtu.be/test", true}, + {"https://reddit.com/r/test/comments/123", true}, + {"https://www.reddit.com/r/test/comments/123", true}, + {"https://example.com/article", false}, + {"https://news.ycombinator.com/item?id=123", false}, + {"https://kite.kagi.com/search?q=test", false}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + result := isOEmbedProvider(tt.url) + assert.Equal(t, tt.expected, result, "URL: %s", tt.url) + }) + } +} + +func TestIsSupported(t *testing.T) { + tests := []struct { + url string + expected bool + }{ + {"https://example.com", true}, + {"http://example.com", true}, + {"https://news.site.com/article", true}, + {"ftp://example.com", false}, + {"not-a-url", false}, + {"", false}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + result := isSupported(tt.url) + assert.Equal(t, tt.expected, result, "URL: %s", tt.url) + }) + } +} + +func TestGetAttr(t *testing.T) { + html := `` + doc, err := parseOpenGraph(html) + require.NoError(t, err) + + // This is a simple test to verify the helper function works + // The actual usage is tested in the parseOpenGraph tests + assert.NotNil(t, doc) +} diff --git a/internal/core/unfurl/providers.go b/internal/core/unfurl/providers.go new file mode 100644 index 0000000..d968a9d --- /dev/null +++ b/internal/core/unfurl/providers.go @@ -0,0 +1,436 @@ +package unfurl + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/net/html" +) + +// Provider configuration +var oEmbedEndpoints = map[string]string{ + "streamable.com": "https://api.streamable.com/oembed", + "youtube.com": "https://www.youtube.com/oembed", + "youtu.be": "https://www.youtube.com/oembed", + "reddit.com": "https://www.reddit.com/oembed", +} + +// oEmbedResponse represents a standard oEmbed response +type oEmbedResponse struct { + ThumbnailURL string `json:"thumbnail_url"` + Version string `json:"version"` + Title string `json:"title"` + AuthorName string `json:"author_name"` + ProviderName string `json:"provider_name"` + ProviderURL string `json:"provider_url"` + Type string `json:"type"` + HTML string `json:"html"` + Description string `json:"description"` + ThumbnailWidth int `json:"thumbnail_width"` + ThumbnailHeight int `json:"thumbnail_height"` + Width int `json:"width"` + Height int `json:"height"` +} + +// extractDomain extracts the domain from a URL +func extractDomain(urlStr string) string { + parsed, err := url.Parse(urlStr) + if err != nil { + return "" + } + // Remove www. prefix + domain := strings.TrimPrefix(parsed.Host, "www.") + return domain +} + +// isSupported checks if this is a valid HTTP/HTTPS URL +func isSupported(urlStr string) bool { + parsed, err := url.Parse(urlStr) + if err != nil { + return false + } + scheme := strings.ToLower(parsed.Scheme) + return scheme == "http" || scheme == "https" +} + +// isOEmbedProvider checks if we have an oEmbed endpoint for this URL +func isOEmbedProvider(urlStr string) bool { + domain := extractDomain(urlStr) + _, exists := oEmbedEndpoints[domain] + return exists +} + +// fetchOEmbed fetches oEmbed data from the provider +func fetchOEmbed(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*oEmbedResponse, error) { + domain := extractDomain(urlStr) + endpoint, exists := oEmbedEndpoints[domain] + if !exists { + return nil, fmt.Errorf("no oEmbed endpoint for domain: %s", domain) + } + + // Build oEmbed request URL + oembedURL := fmt.Sprintf("%s?url=%s&format=json", endpoint, url.QueryEscape(urlStr)) + + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", oembedURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create oEmbed request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + // Create HTTP client with timeout + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch oEmbed data: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("oEmbed endpoint returned status %d", resp.StatusCode) + } + + // Parse JSON response + var oembed oEmbedResponse + if err := json.NewDecoder(resp.Body).Decode(&oembed); err != nil { + return nil, fmt.Errorf("failed to parse oEmbed response: %w", err) + } + + return &oembed, nil +} + +// mapOEmbedToResult converts oEmbed response to UnfurlResult +func mapOEmbedToResult(oembed *oEmbedResponse, originalURL string) *UnfurlResult { + result := &UnfurlResult{ + URI: originalURL, + Title: oembed.Title, + Description: oembed.Description, + ThumbnailURL: oembed.ThumbnailURL, + Provider: strings.ToLower(oembed.ProviderName), + Domain: extractDomain(originalURL), + Width: oembed.Width, + Height: oembed.Height, + } + + // Map oEmbed type to our embedType + switch oembed.Type { + case "video": + result.Type = "video" + case "photo": + result.Type = "image" + default: + result.Type = "article" + } + + // If no description but we have author name, use that + if result.Description == "" && oembed.AuthorName != "" { + result.Description = fmt.Sprintf("By %s", oembed.AuthorName) + } + + return result +} + +// openGraphData represents OpenGraph metadata extracted from HTML +type openGraphData struct { + Title string + Description string + Image string + URL string +} + +// fetchOpenGraph fetches OpenGraph metadata from a URL +func fetchOpenGraph(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*UnfurlResult, error) { + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + // Create HTTP client with timeout + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch URL: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request returned status %d", resp.StatusCode) + } + + // Read response body (limit to 10MB to prevent abuse) + limitedReader := io.LimitReader(resp.Body, 10*1024*1024) + body, err := io.ReadAll(limitedReader) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + // Parse OpenGraph metadata + og, err := parseOpenGraph(string(body)) + if err != nil { + return nil, fmt.Errorf("failed to parse OpenGraph metadata: %w", err) + } + + // Build UnfurlResult + result := &UnfurlResult{ + Type: "article", // Default type for OpenGraph + URI: urlStr, + Title: og.Title, + Description: og.Description, + ThumbnailURL: og.Image, + Provider: "opengraph", + Domain: extractDomain(urlStr), + } + + // Use og:url if available and valid + if og.URL != "" { + result.URI = og.URL + } + + return result, nil +} + +// parseOpenGraph extracts OpenGraph metadata from HTML +func parseOpenGraph(htmlContent string) (*openGraphData, error) { + og := &openGraphData{} + doc, err := html.Parse(strings.NewReader(htmlContent)) + if err != nil { + // Try best-effort parsing even with invalid HTML + return og, nil + } + + // Extract OpenGraph tags and fallbacks + var pageTitle string + var metaDescription string + + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "meta": + property := getAttr(n, "property") + name := getAttr(n, "name") + content := getAttr(n, "content") + + // OpenGraph tags + if strings.HasPrefix(property, "og:") { + switch property { + case "og:title": + if og.Title == "" { + og.Title = content + } + case "og:description": + if og.Description == "" { + og.Description = content + } + case "og:image": + if og.Image == "" { + og.Image = content + } + case "og:url": + if og.URL == "" { + og.URL = content + } + } + } + + // Fallback meta tags + if name == "description" && metaDescription == "" { + metaDescription = content + } + + case "title": + if pageTitle == "" && n.FirstChild != nil { + pageTitle = n.FirstChild.Data + } + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + + traverse(doc) + + // Apply fallbacks + if og.Title == "" { + og.Title = pageTitle + } + if og.Description == "" { + og.Description = metaDescription + } + + return og, nil +} + +// getAttr gets an attribute value from an HTML node +func getAttr(n *html.Node, key string) string { + for _, attr := range n.Attr { + if attr.Key == key { + return attr.Val + } + } + return "" +} + +// fetchKagiKite handles special unfurling for Kagi Kite news pages +// Kagi Kite pages use client-side rendering, so og:image tags aren't available at SSR time +// Instead, we parse the HTML to extract the story image from the page content +func fetchKagiKite(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*UnfurlResult, error) { + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + // Create HTTP client with timeout + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch URL: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) + } + + // Limit response size to 10MB + limitedReader := io.LimitReader(resp.Body, 10*1024*1024) + + // Parse HTML + doc, err := html.Parse(limitedReader) + if err != nil { + return nil, fmt.Errorf("failed to parse HTML: %w", err) + } + + result := &UnfurlResult{ + Type: "article", + URI: urlStr, + Domain: "kite.kagi.com", + Provider: "kagi", + } + + // First try OpenGraph tags (in case they get added in the future) + var findOG func(*html.Node) + findOG = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "meta" { + var property, content string + for _, attr := range n.Attr { + if attr.Key == "property" { + property = attr.Val + } else if attr.Key == "content" { + content = attr.Val + } + } + + switch property { + case "og:title": + if result.Title == "" { + result.Title = content + } + case "og:description": + if result.Description == "" { + result.Description = content + } + case "og:image": + if result.ThumbnailURL == "" { + result.ThumbnailURL = content + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + findOG(c) + } + } + findOG(doc) + + // Fallback: Extract from page content + // Look for images with kagiproxy.com URLs (Kagi's image proxy) + // Note: Skip the first image as it's often a shared header/logo + if result.ThumbnailURL == "" { + var images []struct { + url string + alt string + } + + var findImg func(*html.Node) + findImg = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "img" { + for _, attr := range n.Attr { + if attr.Key == "src" && strings.Contains(attr.Val, "kagiproxy.com") { + // Get alt text if available + var altText string + for _, a := range n.Attr { + if a.Key == "alt" { + altText = a.Val + break + } + } + images = append(images, struct { + url string + alt string + }{url: attr.Val, alt: altText}) + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + findImg(c) + } + } + findImg(doc) + + // Skip first image (often shared header/logo), use second if available + if len(images) > 1 { + result.ThumbnailURL = images[1].url + if result.Description == "" && images[1].alt != "" { + result.Description = images[1].alt + } + } else if len(images) == 1 { + // Only one image found, use it + result.ThumbnailURL = images[0].url + if result.Description == "" && images[0].alt != "" { + result.Description = images[0].alt + } + } + } + + // Fallback to