diff --git a/internal/core/unfurl/circuit_breaker.go b/internal/core/unfurl/circuit_breaker.go new file mode 100644 index 0000000..54815f9 --- /dev/null +++ b/internal/core/unfurl/circuit_breaker.go @@ -0,0 +1,200 @@ +package unfurl + +import ( + "fmt" + "log" + "sync" + "time" +) + +// circuitState represents the state of a circuit breaker +type circuitState int + +const ( + stateClosed circuitState = iota // Normal operation + stateOpen // Circuit is open (provider failing) + stateHalfOpen // Testing if provider recovered +) + +// circuitBreaker tracks failures per provider and stops trying failing providers +type circuitBreaker struct { + failures map[string]int + lastFailure map[string]time.Time + state map[string]circuitState + lastStateLog map[string]time.Time + failureThreshold int + openDuration time.Duration + mu sync.RWMutex +} + +// newCircuitBreaker creates a circuit breaker with default settings +func newCircuitBreaker() *circuitBreaker { + return &circuitBreaker{ + failureThreshold: 3, // Open after 3 consecutive failures + openDuration: 5 * time.Minute, // Keep open for 5 minutes + failures: make(map[string]int), + lastFailure: make(map[string]time.Time), + state: make(map[string]circuitState), + lastStateLog: make(map[string]time.Time), + } +} + +// canAttempt checks if we should attempt to call this provider +// Returns true if circuit is closed or half-open (ready to retry) +func (cb *circuitBreaker) canAttempt(provider string) (bool, error) { + cb.mu.RLock() + defer cb.mu.RUnlock() + + state := cb.getState(provider) + + switch state { + case stateClosed: + return true, nil + case stateOpen: + // Check if we should transition to half-open + lastFail := cb.lastFailure[provider] + if time.Since(lastFail) > cb.openDuration { + // Transition to half-open (allow one retry) + cb.mu.RUnlock() + cb.mu.Lock() + cb.state[provider] = stateHalfOpen + cb.logStateChange(provider, stateHalfOpen) + cb.mu.Unlock() + cb.mu.RLock() + return true, nil + } + // Still in open period + failCount := cb.failures[provider] + nextRetry := lastFail.Add(cb.openDuration) + return false, fmt.Errorf( + "circuit breaker open for provider '%s' (failures: %d, next retry: %s)", + provider, + failCount, + nextRetry.Format("15:04:05"), + ) + case stateHalfOpen: + return true, nil + default: + return true, nil + } +} + +// recordSuccess records a successful unfurl, resetting failure count +func (cb *circuitBreaker) recordSuccess(provider string) { + cb.mu.Lock() + defer cb.mu.Unlock() + + oldState := cb.getState(provider) + + // Reset failure tracking + delete(cb.failures, provider) + delete(cb.lastFailure, provider) + cb.state[provider] = stateClosed + + // Log recovery if we were in a failure state + if oldState != stateClosed { + cb.logStateChange(provider, stateClosed) + } +} + +// recordFailure records a failed unfurl attempt +func (cb *circuitBreaker) recordFailure(provider string, err error) { + cb.mu.Lock() + defer cb.mu.Unlock() + + // Increment failure count + cb.failures[provider]++ + cb.lastFailure[provider] = time.Now() + + failCount := cb.failures[provider] + + // Check if we should open the circuit + if failCount >= cb.failureThreshold { + oldState := cb.getState(provider) + cb.state[provider] = stateOpen + if oldState != stateOpen { + log.Printf( + "[UNFURL-CIRCUIT] Opening circuit for provider '%s' after %d consecutive failures. Last error: %v", + provider, + failCount, + err, + ) + cb.lastStateLog[provider] = time.Now() + } + } else { + log.Printf( + "[UNFURL-CIRCUIT] Failure %d/%d for provider '%s': %v", + failCount, + cb.failureThreshold, + provider, + err, + ) + } +} + +// getState returns the current state (must be called with lock held) +func (cb *circuitBreaker) getState(provider string) circuitState { + if state, exists := cb.state[provider]; exists { + return state + } + return stateClosed +} + +// logStateChange logs state transitions (must be called with lock held) +// Debounced to avoid log spam (max once per minute per provider) +func (cb *circuitBreaker) logStateChange(provider string, newState circuitState) { + lastLog, exists := cb.lastStateLog[provider] + if exists && time.Since(lastLog) < time.Minute { + return // Don't spam logs + } + + var stateStr string + switch newState { + case stateClosed: + stateStr = "CLOSED (recovered)" + case stateOpen: + stateStr = "OPEN (failing)" + case stateHalfOpen: + stateStr = "HALF-OPEN (testing)" + } + + log.Printf("[UNFURL-CIRCUIT] Circuit for provider '%s' is now %s", provider, stateStr) + cb.lastStateLog[provider] = time.Now() +} + +// getStats returns current circuit breaker stats (for debugging/monitoring) +func (cb *circuitBreaker) getStats() map[string]interface{} { + cb.mu.RLock() + defer cb.mu.RUnlock() + + stats := make(map[string]interface{}) + + // Collect all providers with any activity (state, failures, or both) + providers := make(map[string]bool) + for provider := range cb.state { + providers[provider] = true + } + for provider := range cb.failures { + providers[provider] = true + } + + for provider := range providers { + state := cb.getState(provider) + var stateStr string + switch state { + case stateClosed: + stateStr = "closed" + case stateOpen: + stateStr = "open" + case stateHalfOpen: + stateStr = "half-open" + } + + stats[provider] = map[string]interface{}{ + "state": stateStr, + "failures": cb.failures[provider], + "last_failure": cb.lastFailure[provider], + } + } + return stats +} diff --git a/internal/core/unfurl/circuit_breaker_test.go b/internal/core/unfurl/circuit_breaker_test.go new file mode 100644 index 0000000..34cf01b --- /dev/null +++ b/internal/core/unfurl/circuit_breaker_test.go @@ -0,0 +1,175 @@ +package unfurl + +import ( + "fmt" + "testing" + "time" +) + +func TestCircuitBreaker_Basic(t *testing.T) { + cb := newCircuitBreaker() + + provider := "test-provider" + + // Should start closed (allow attempts) + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to be closed initially, but got error: %v", err) + } + + // Record success + cb.recordSuccess(provider) + canAttempt, _ = cb.canAttempt(provider) + if !canAttempt { + t.Error("Expected circuit to remain closed after success") + } +} + +func TestCircuitBreaker_OpensAfterFailures(t *testing.T) { + cb := newCircuitBreaker() + provider := "failing-provider" + + // Record failures up to threshold + for i := 0; i < cb.failureThreshold; i++ { + cb.recordFailure(provider, fmt.Errorf("test error %d", i)) + } + + // Circuit should now be open + canAttempt, err := cb.canAttempt(provider) + if canAttempt { + t.Error("Expected circuit to be open after threshold failures") + } + if err == nil { + t.Error("Expected error when circuit is open") + } +} + +func TestCircuitBreaker_RecoveryAfterSuccess(t *testing.T) { + cb := newCircuitBreaker() + provider := "recovery-provider" + + // Record some failures + cb.recordFailure(provider, fmt.Errorf("error 1")) + cb.recordFailure(provider, fmt.Errorf("error 2")) + + // Record success - should reset failure count + cb.recordSuccess(provider) + + // Should be able to attempt again + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to be closed after success, but got error: %v", err) + } + + // Failure count should be reset + if count := cb.failures[provider]; count != 0 { + t.Errorf("Expected failure count to be reset to 0, got %d", count) + } +} + +func TestCircuitBreaker_HalfOpenTransition(t *testing.T) { + cb := newCircuitBreaker() + cb.openDuration = 100 * time.Millisecond // Short duration for testing + provider := "half-open-provider" + + // Open the circuit + for i := 0; i < cb.failureThreshold; i++ { + cb.recordFailure(provider, fmt.Errorf("error %d", i)) + } + + // Should be open + canAttempt, _ := cb.canAttempt(provider) + if canAttempt { + t.Error("Expected circuit to be open") + } + + // Wait for open duration + time.Sleep(150 * time.Millisecond) + + // Should transition to half-open and allow one attempt + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to transition to half-open after duration, but got error: %v", err) + } + + // State should be half-open + cb.mu.RLock() + state := cb.state[provider] + cb.mu.RUnlock() + + if state != stateHalfOpen { + t.Errorf("Expected state to be half-open, got %v", state) + } +} + +func TestCircuitBreaker_MultipleProviders(t *testing.T) { + cb := newCircuitBreaker() + + // Open circuit for provider A + for i := 0; i < cb.failureThreshold; i++ { + cb.recordFailure("providerA", fmt.Errorf("error")) + } + + // Provider A should be blocked + canAttemptA, _ := cb.canAttempt("providerA") + if canAttemptA { + t.Error("Expected providerA circuit to be open") + } + + // Provider B should still be open (independent circuits) + canAttemptB, err := cb.canAttempt("providerB") + if !canAttemptB { + t.Errorf("Expected providerB circuit to be closed, but got error: %v", err) + } +} + +func TestCircuitBreaker_GetStats(t *testing.T) { + cb := newCircuitBreaker() + + // Record some activity + cb.recordFailure("provider1", fmt.Errorf("error 1")) + cb.recordFailure("provider1", fmt.Errorf("error 2")) + + stats := cb.getStats() + + // Should have stats for providers with failures + if providerStats, ok := stats["provider1"]; !ok { + t.Error("Expected stats for provider1") + } else { + // Check that failure count is tracked + statsMap := providerStats.(map[string]interface{}) + if failures, ok := statsMap["failures"].(int); !ok || failures != 2 { + t.Errorf("Expected 2 failures for provider1, got %v", statsMap["failures"]) + } + } + + // Provider that succeeds is cleaned up from state + cb.recordSuccess("provider2") + _ = cb.getStats() + // Provider2 should not be in stats (or have state "closed" with 0 failures) +} + +func TestCircuitBreaker_FailureThresholdExact(t *testing.T) { + cb := newCircuitBreaker() + provider := "exact-threshold-provider" + + // Record failures just below threshold + for i := 0; i < cb.failureThreshold-1; i++ { + cb.recordFailure(provider, fmt.Errorf("error %d", i)) + } + + // Should still be closed + canAttempt, err := cb.canAttempt(provider) + if !canAttempt { + t.Errorf("Expected circuit to be closed below threshold, but got error: %v", err) + } + + // One more failure should open it + cb.recordFailure(provider, fmt.Errorf("final error")) + + // Should now be open + canAttempt, _ = cb.canAttempt(provider) + if canAttempt { + t.Error("Expected circuit to be open at threshold") + } +} diff --git a/internal/core/unfurl/kagi_test.go b/internal/core/unfurl/kagi_test.go new file mode 100644 index 0000000..f5e79a1 --- /dev/null +++ b/internal/core/unfurl/kagi_test.go @@ -0,0 +1,202 @@ +package unfurl + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFetchKagiKite_Success(t *testing.T) { + // Mock Kagi HTML response + mockHTML := ` + + + FAA orders 10% flight cuts at 40 airports - Kagi News + + + + + Airport runway + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + require.NoError(t, err) + assert.Equal(t, "article", result.Type) + assert.Equal(t, "FAA orders 10% flight cuts", result.Title) + assert.Equal(t, "Flight restrictions announced", result.Description) + assert.Contains(t, result.ThumbnailURL, "kagiproxy.com") + assert.Equal(t, "kagi", result.Provider) + assert.Equal(t, "kite.kagi.com", result.Domain) +} + +func TestFetchKagiKite_NoImage(t *testing.T) { + mockHTML := ` + +Test Story +

No images here

+` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "no image found") +} + +func TestFetchKagiKite_FallbackToTitle(t *testing.T) { + mockHTML := ` + +Fallback Title + + + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + require.NoError(t, err) + assert.Equal(t, "Fallback Title", result.Title) + assert.Contains(t, result.ThumbnailURL, "kagiproxy.com") +} + +func TestFetchKagiKite_ImageWithAltText(t *testing.T) { + mockHTML := ` + +News Story + + This is the alt text description + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + require.NoError(t, err) + assert.Equal(t, "News Story", result.Title) + assert.Equal(t, "This is the alt text description", result.Description) + assert.Contains(t, result.ThumbnailURL, "kagiproxy.com") +} + +func TestFetchKagiKite_HTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "HTTP 404") +} + +func TestFetchKagiKite_Timeout(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 100*time.Millisecond, "TestBot/1.0") + + assert.Error(t, err) + assert.Nil(t, result) +} + +func TestFetchKagiKite_MultipleImages_PicksSecond(t *testing.T) { + mockHTML := ` + +Story with multiple images + + First image (header/logo) + Second image + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + require.NoError(t, err) + // We skip the first image (often a header/logo) and use the second + assert.Contains(t, result.ThumbnailURL, "second456") + assert.Equal(t, "Second image", result.Description) +} + +func TestFetchKagiKite_OnlyNonKagiImages_NoMatch(t *testing.T) { + mockHTML := ` + +Story with non-Kagi images + + External image + +` + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(mockHTML)) + })) + defer server.Close() + + ctx := context.Background() + + result, err := fetchKagiKite(ctx, server.URL, 5*time.Second, "TestBot/1.0") + + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "no image found") +} diff --git a/internal/core/unfurl/opengraph_test.go b/internal/core/unfurl/opengraph_test.go new file mode 100644 index 0000000..cb490f6 --- /dev/null +++ b/internal/core/unfurl/opengraph_test.go @@ -0,0 +1,269 @@ +package unfurl + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseOpenGraph_ValidTags(t *testing.T) { + html := ` + + + + + + + + + +

Some content

+ + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Equal(t, "Test Article Title", og.Title) + assert.Equal(t, "This is a test description", og.Description) + assert.Equal(t, "https://example.com/image.jpg", og.Image) + assert.Equal(t, "https://example.com/canonical", og.URL) +} + +func TestParseOpenGraph_MissingImage(t *testing.T) { + html := ` + + + + + + + + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Equal(t, "Article Without Image", og.Title) + assert.Equal(t, "No image tag", og.Description) + assert.Empty(t, og.Image, "Image should be empty when not provided") +} + +func TestParseOpenGraph_FallbackToTitle(t *testing.T) { + html := ` + + + + Page Title Fallback + + + + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Equal(t, "Page Title Fallback", og.Title, "Should fall back to ") + assert.Equal(t, "Meta description fallback", og.Description, "Should fall back to meta description") +} + +func TestParseOpenGraph_PreferOpenGraphOverFallback(t *testing.T) { + html := ` +<!DOCTYPE html> +<html> +<head> + <title>Page Title + + + + + + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Equal(t, "OpenGraph Title", og.Title, "Should prefer og:title") + assert.Equal(t, "OpenGraph Description", og.Description, "Should prefer og:description") +} + +func TestParseOpenGraph_MalformedHTML(t *testing.T) { + html := ` + + + + + + + + + +` + + og, err := parseOpenGraph(html) + require.NoError(t, err) + + assert.Empty(t, og.Title) + assert.Empty(t, og.Description) + assert.Empty(t, og.Image) +} + +func TestFetchOpenGraph_Success(t *testing.T) { + // Create test server with OpenGraph metadata + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Contains(t, r.Header.Get("User-Agent"), "CovesBot") + + html := ` + + + + + + + + +

Article content

+ +` + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(html)) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + require.NoError(t, err) + require.NotNil(t, result) + + assert.Equal(t, "Test News Article", result.Title) + assert.Equal(t, "Breaking news story", result.Description) + assert.Equal(t, "https://example.com/news.jpg", result.ThumbnailURL) + assert.Equal(t, "article", result.Type) + assert.Equal(t, "opengraph", result.Provider) +} + +func TestFetchOpenGraph_HTTPError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + require.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "404") +} + +func TestFetchOpenGraph_Timeout(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(2 * time.Second) + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 100*time.Millisecond, "CovesBot/1.0") + require.Error(t, err) + assert.Nil(t, result) +} + +func TestFetchOpenGraph_NoMetadata(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + html := `

No metadata

` + w.Header().Set("Content-Type", "text/html") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(html)) + })) + defer server.Close() + + ctx := context.Background() + result, err := fetchOpenGraph(ctx, server.URL, 10*time.Second, "CovesBot/1.0") + require.NoError(t, err) + require.NotNil(t, result) + + // Should still return a result with domain + assert.Equal(t, "article", result.Type) + assert.Equal(t, "opengraph", result.Provider) + assert.NotEmpty(t, result.Domain) +} + +func TestIsOEmbedProvider(t *testing.T) { + tests := []struct { + url string + expected bool + }{ + {"https://streamable.com/abc123", true}, + {"https://www.youtube.com/watch?v=test", true}, + {"https://youtu.be/test", true}, + {"https://reddit.com/r/test/comments/123", true}, + {"https://www.reddit.com/r/test/comments/123", true}, + {"https://example.com/article", false}, + {"https://news.ycombinator.com/item?id=123", false}, + {"https://kite.kagi.com/search?q=test", false}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + result := isOEmbedProvider(tt.url) + assert.Equal(t, tt.expected, result, "URL: %s", tt.url) + }) + } +} + +func TestIsSupported(t *testing.T) { + tests := []struct { + url string + expected bool + }{ + {"https://example.com", true}, + {"http://example.com", true}, + {"https://news.site.com/article", true}, + {"ftp://example.com", false}, + {"not-a-url", false}, + {"", false}, + } + + for _, tt := range tests { + t.Run(tt.url, func(t *testing.T) { + result := isSupported(tt.url) + assert.Equal(t, tt.expected, result, "URL: %s", tt.url) + }) + } +} + +func TestGetAttr(t *testing.T) { + html := `` + doc, err := parseOpenGraph(html) + require.NoError(t, err) + + // This is a simple test to verify the helper function works + // The actual usage is tested in the parseOpenGraph tests + assert.NotNil(t, doc) +} diff --git a/internal/core/unfurl/providers.go b/internal/core/unfurl/providers.go new file mode 100644 index 0000000..d968a9d --- /dev/null +++ b/internal/core/unfurl/providers.go @@ -0,0 +1,436 @@ +package unfurl + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/net/html" +) + +// Provider configuration +var oEmbedEndpoints = map[string]string{ + "streamable.com": "https://api.streamable.com/oembed", + "youtube.com": "https://www.youtube.com/oembed", + "youtu.be": "https://www.youtube.com/oembed", + "reddit.com": "https://www.reddit.com/oembed", +} + +// oEmbedResponse represents a standard oEmbed response +type oEmbedResponse struct { + ThumbnailURL string `json:"thumbnail_url"` + Version string `json:"version"` + Title string `json:"title"` + AuthorName string `json:"author_name"` + ProviderName string `json:"provider_name"` + ProviderURL string `json:"provider_url"` + Type string `json:"type"` + HTML string `json:"html"` + Description string `json:"description"` + ThumbnailWidth int `json:"thumbnail_width"` + ThumbnailHeight int `json:"thumbnail_height"` + Width int `json:"width"` + Height int `json:"height"` +} + +// extractDomain extracts the domain from a URL +func extractDomain(urlStr string) string { + parsed, err := url.Parse(urlStr) + if err != nil { + return "" + } + // Remove www. prefix + domain := strings.TrimPrefix(parsed.Host, "www.") + return domain +} + +// isSupported checks if this is a valid HTTP/HTTPS URL +func isSupported(urlStr string) bool { + parsed, err := url.Parse(urlStr) + if err != nil { + return false + } + scheme := strings.ToLower(parsed.Scheme) + return scheme == "http" || scheme == "https" +} + +// isOEmbedProvider checks if we have an oEmbed endpoint for this URL +func isOEmbedProvider(urlStr string) bool { + domain := extractDomain(urlStr) + _, exists := oEmbedEndpoints[domain] + return exists +} + +// fetchOEmbed fetches oEmbed data from the provider +func fetchOEmbed(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*oEmbedResponse, error) { + domain := extractDomain(urlStr) + endpoint, exists := oEmbedEndpoints[domain] + if !exists { + return nil, fmt.Errorf("no oEmbed endpoint for domain: %s", domain) + } + + // Build oEmbed request URL + oembedURL := fmt.Sprintf("%s?url=%s&format=json", endpoint, url.QueryEscape(urlStr)) + + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", oembedURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create oEmbed request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + // Create HTTP client with timeout + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch oEmbed data: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("oEmbed endpoint returned status %d", resp.StatusCode) + } + + // Parse JSON response + var oembed oEmbedResponse + if err := json.NewDecoder(resp.Body).Decode(&oembed); err != nil { + return nil, fmt.Errorf("failed to parse oEmbed response: %w", err) + } + + return &oembed, nil +} + +// mapOEmbedToResult converts oEmbed response to UnfurlResult +func mapOEmbedToResult(oembed *oEmbedResponse, originalURL string) *UnfurlResult { + result := &UnfurlResult{ + URI: originalURL, + Title: oembed.Title, + Description: oembed.Description, + ThumbnailURL: oembed.ThumbnailURL, + Provider: strings.ToLower(oembed.ProviderName), + Domain: extractDomain(originalURL), + Width: oembed.Width, + Height: oembed.Height, + } + + // Map oEmbed type to our embedType + switch oembed.Type { + case "video": + result.Type = "video" + case "photo": + result.Type = "image" + default: + result.Type = "article" + } + + // If no description but we have author name, use that + if result.Description == "" && oembed.AuthorName != "" { + result.Description = fmt.Sprintf("By %s", oembed.AuthorName) + } + + return result +} + +// openGraphData represents OpenGraph metadata extracted from HTML +type openGraphData struct { + Title string + Description string + Image string + URL string +} + +// fetchOpenGraph fetches OpenGraph metadata from a URL +func fetchOpenGraph(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*UnfurlResult, error) { + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + // Create HTTP client with timeout + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch URL: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request returned status %d", resp.StatusCode) + } + + // Read response body (limit to 10MB to prevent abuse) + limitedReader := io.LimitReader(resp.Body, 10*1024*1024) + body, err := io.ReadAll(limitedReader) + if err != nil { + return nil, fmt.Errorf("failed to read response body: %w", err) + } + + // Parse OpenGraph metadata + og, err := parseOpenGraph(string(body)) + if err != nil { + return nil, fmt.Errorf("failed to parse OpenGraph metadata: %w", err) + } + + // Build UnfurlResult + result := &UnfurlResult{ + Type: "article", // Default type for OpenGraph + URI: urlStr, + Title: og.Title, + Description: og.Description, + ThumbnailURL: og.Image, + Provider: "opengraph", + Domain: extractDomain(urlStr), + } + + // Use og:url if available and valid + if og.URL != "" { + result.URI = og.URL + } + + return result, nil +} + +// parseOpenGraph extracts OpenGraph metadata from HTML +func parseOpenGraph(htmlContent string) (*openGraphData, error) { + og := &openGraphData{} + doc, err := html.Parse(strings.NewReader(htmlContent)) + if err != nil { + // Try best-effort parsing even with invalid HTML + return og, nil + } + + // Extract OpenGraph tags and fallbacks + var pageTitle string + var metaDescription string + + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "meta": + property := getAttr(n, "property") + name := getAttr(n, "name") + content := getAttr(n, "content") + + // OpenGraph tags + if strings.HasPrefix(property, "og:") { + switch property { + case "og:title": + if og.Title == "" { + og.Title = content + } + case "og:description": + if og.Description == "" { + og.Description = content + } + case "og:image": + if og.Image == "" { + og.Image = content + } + case "og:url": + if og.URL == "" { + og.URL = content + } + } + } + + // Fallback meta tags + if name == "description" && metaDescription == "" { + metaDescription = content + } + + case "title": + if pageTitle == "" && n.FirstChild != nil { + pageTitle = n.FirstChild.Data + } + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + + traverse(doc) + + // Apply fallbacks + if og.Title == "" { + og.Title = pageTitle + } + if og.Description == "" { + og.Description = metaDescription + } + + return og, nil +} + +// getAttr gets an attribute value from an HTML node +func getAttr(n *html.Node, key string) string { + for _, attr := range n.Attr { + if attr.Key == key { + return attr.Val + } + } + return "" +} + +// fetchKagiKite handles special unfurling for Kagi Kite news pages +// Kagi Kite pages use client-side rendering, so og:image tags aren't available at SSR time +// Instead, we parse the HTML to extract the story image from the page content +func fetchKagiKite(ctx context.Context, urlStr string, timeout time.Duration, userAgent string) (*UnfurlResult, error) { + // Create HTTP request + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("User-Agent", userAgent) + + // Create HTTP client with timeout + client := &http.Client{Timeout: timeout} + resp, err := client.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to fetch URL: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) + } + + // Limit response size to 10MB + limitedReader := io.LimitReader(resp.Body, 10*1024*1024) + + // Parse HTML + doc, err := html.Parse(limitedReader) + if err != nil { + return nil, fmt.Errorf("failed to parse HTML: %w", err) + } + + result := &UnfurlResult{ + Type: "article", + URI: urlStr, + Domain: "kite.kagi.com", + Provider: "kagi", + } + + // First try OpenGraph tags (in case they get added in the future) + var findOG func(*html.Node) + findOG = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "meta" { + var property, content string + for _, attr := range n.Attr { + if attr.Key == "property" { + property = attr.Val + } else if attr.Key == "content" { + content = attr.Val + } + } + + switch property { + case "og:title": + if result.Title == "" { + result.Title = content + } + case "og:description": + if result.Description == "" { + result.Description = content + } + case "og:image": + if result.ThumbnailURL == "" { + result.ThumbnailURL = content + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + findOG(c) + } + } + findOG(doc) + + // Fallback: Extract from page content + // Look for images with kagiproxy.com URLs (Kagi's image proxy) + // Note: Skip the first image as it's often a shared header/logo + if result.ThumbnailURL == "" { + var images []struct { + url string + alt string + } + + var findImg func(*html.Node) + findImg = func(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "img" { + for _, attr := range n.Attr { + if attr.Key == "src" && strings.Contains(attr.Val, "kagiproxy.com") { + // Get alt text if available + var altText string + for _, a := range n.Attr { + if a.Key == "alt" { + altText = a.Val + break + } + } + images = append(images, struct { + url string + alt string + }{url: attr.Val, alt: altText}) + break + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + findImg(c) + } + } + findImg(doc) + + // Skip first image (often shared header/logo), use second if available + if len(images) > 1 { + result.ThumbnailURL = images[1].url + if result.Description == "" && images[1].alt != "" { + result.Description = images[1].alt + } + } else if len(images) == 1 { + // Only one image found, use it + result.ThumbnailURL = images[0].url + if result.Description == "" && images[0].alt != "" { + result.Description = images[0].alt + } + } + } + + // Fallback to tag if og:title not found + if result.Title == "" { + var findTitle func(*html.Node) string + findTitle = func(n *html.Node) string { + if n.Type == html.ElementNode && n.Data == "title" { + if n.FirstChild != nil && n.FirstChild.Type == html.TextNode { + return n.FirstChild.Data + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + if title := findTitle(c); title != "" { + return title + } + } + return "" + } + result.Title = findTitle(doc) + } + + // If still no image, return error + if result.ThumbnailURL == "" { + return nil, fmt.Errorf("no image found in Kagi page") + } + + return result, nil +} diff --git a/internal/core/unfurl/service.go b/internal/core/unfurl/service.go new file mode 100644 index 0000000..748d92c --- /dev/null +++ b/internal/core/unfurl/service.go @@ -0,0 +1,170 @@ +package unfurl + +import ( + "context" + "fmt" + "log" + "time" +) + +// Service handles URL unfurling with caching +type Service interface { + UnfurlURL(ctx context.Context, url string) (*UnfurlResult, error) + IsSupported(url string) bool +} + +type service struct { + repo Repository + circuitBreaker *circuitBreaker + userAgent string + timeout time.Duration + cacheTTL time.Duration +} + +// NewService creates a new unfurl service +func NewService(repo Repository, opts ...ServiceOption) Service { + s := &service{ + repo: repo, + timeout: 10 * time.Second, + userAgent: "CovesBot/1.0 (+https://coves.social)", + cacheTTL: 24 * time.Hour, + circuitBreaker: newCircuitBreaker(), + } + + for _, opt := range opts { + opt(s) + } + + return s +} + +// ServiceOption configures the service +type ServiceOption func(*service) + +// WithTimeout sets the HTTP timeout for oEmbed requests +func WithTimeout(timeout time.Duration) ServiceOption { + return func(s *service) { + s.timeout = timeout + } +} + +// WithUserAgent sets the User-Agent header for oEmbed requests +func WithUserAgent(userAgent string) ServiceOption { + return func(s *service) { + s.userAgent = userAgent + } +} + +// WithCacheTTL sets the cache TTL +func WithCacheTTL(ttl time.Duration) ServiceOption { + return func(s *service) { + s.cacheTTL = ttl + } +} + +// IsSupported returns true if we can unfurl this URL +func (s *service) IsSupported(url string) bool { + return isSupported(url) +} + +// UnfurlURL fetches metadata for a URL (with caching) +func (s *service) UnfurlURL(ctx context.Context, urlStr string) (*UnfurlResult, error) { + // 1. Check cache first + cached, err := s.repo.Get(ctx, urlStr) + if err == nil && cached != nil { + log.Printf("[UNFURL] Cache hit for %s (provider: %s)", urlStr, cached.Provider) + return cached, nil + } + + // 2. Check if we support this URL + if !isSupported(urlStr) { + return nil, fmt.Errorf("unsupported URL: %s", urlStr) + } + + var result *UnfurlResult + domain := extractDomain(urlStr) + + // 3. Smart routing: Special handling for Kagi Kite (client-side rendered, no og:image tags) + if domain == "kite.kagi.com" { + provider := "kagi" + + // Check circuit breaker + canAttempt, err := s.circuitBreaker.canAttempt(provider) + if !canAttempt { + log.Printf("[UNFURL] Skipping %s due to circuit breaker: %v", urlStr, err) + return nil, err + } + + log.Printf("[UNFURL] Cache miss for %s, fetching via Kagi parser...", urlStr) + result, err = fetchKagiKite(ctx, urlStr, s.timeout, s.userAgent) + if err != nil { + s.circuitBreaker.recordFailure(provider, err) + return nil, err + } + + s.circuitBreaker.recordSuccess(provider) + + // Cache result + if cacheErr := s.repo.Set(ctx, urlStr, result, s.cacheTTL); cacheErr != nil { + log.Printf("[UNFURL] Warning: failed to cache result: %v", cacheErr) + } + return result, nil + } + + // 4. Check if this is a known oEmbed provider + if isOEmbedProvider(urlStr) { + provider := domain // Use domain as provider name (e.g., "streamable.com", "youtube.com") + + // Check circuit breaker + canAttempt, err := s.circuitBreaker.canAttempt(provider) + if !canAttempt { + log.Printf("[UNFURL] Skipping %s due to circuit breaker: %v", urlStr, err) + return nil, err + } + + log.Printf("[UNFURL] Cache miss for %s, fetching from oEmbed...", urlStr) + + // Fetch from oEmbed provider + oembed, err := fetchOEmbed(ctx, urlStr, s.timeout, s.userAgent) + if err != nil { + s.circuitBreaker.recordFailure(provider, err) + return nil, fmt.Errorf("failed to fetch oEmbed data: %w", err) + } + + s.circuitBreaker.recordSuccess(provider) + + // Convert to UnfurlResult + result = mapOEmbedToResult(oembed, urlStr) + } else { + provider := "opengraph" + + // Check circuit breaker + canAttempt, err := s.circuitBreaker.canAttempt(provider) + if !canAttempt { + log.Printf("[UNFURL] Skipping %s due to circuit breaker: %v", urlStr, err) + return nil, err + } + + log.Printf("[UNFURL] Cache miss for %s, fetching via OpenGraph...", urlStr) + + // Fetch via OpenGraph + result, err = fetchOpenGraph(ctx, urlStr, s.timeout, s.userAgent) + if err != nil { + s.circuitBreaker.recordFailure(provider, err) + return nil, fmt.Errorf("failed to fetch OpenGraph data: %w", err) + } + + s.circuitBreaker.recordSuccess(provider) + } + + // 5. Store in cache + if cacheErr := s.repo.Set(ctx, urlStr, result, s.cacheTTL); cacheErr != nil { + // Log but don't fail - cache is best-effort + log.Printf("[UNFURL] Warning: Failed to cache result for %s: %v", urlStr, cacheErr) + } + + log.Printf("[UNFURL] Successfully unfurled %s (provider: %s, type: %s)", + urlStr, result.Provider, result.Type) + + return result, nil +} diff --git a/internal/core/unfurl/types.go b/internal/core/unfurl/types.go new file mode 100644 index 0000000..c531650 --- /dev/null +++ b/internal/core/unfurl/types.go @@ -0,0 +1,27 @@ +package unfurl + +import "time" + +// UnfurlResult represents the result of unfurling a URL +type UnfurlResult struct { + Type string `json:"type"` // "video", "article", "image", "website" + URI string `json:"uri"` // Original URL + Title string `json:"title"` // Page/video title + Description string `json:"description"` // Page/video description + ThumbnailURL string `json:"thumbnailUrl"` // Preview image URL + Provider string `json:"provider"` // "streamable", "youtube", "reddit" + Domain string `json:"domain"` // Domain of the URL + Width int `json:"width"` // Media width (if applicable) + Height int `json:"height"` // Media height (if applicable) +} + +// CacheEntry represents a cached unfurl result with metadata +type CacheEntry struct { + FetchedAt time.Time `db:"fetched_at"` + ExpiresAt time.Time `db:"expires_at"` + CreatedAt time.Time `db:"created_at"` + ThumbnailURL *string `db:"thumbnail_url"` + URL string `db:"url"` + Provider string `db:"provider"` + Metadata UnfurlResult `db:"metadata"` +}