From 6338881da3f4e791106a53fbee94ba9c509b3cec Mon Sep 17 00:00:00 2001
From: Kieran Klukas <kieran@dunkirk.sh>
Date: Thu, 23 Apr 2026 19:48:37 -0400
Subject: [PATCH] feat: add flaresolver support and jsonld type array

---
 internal/extract/pipeline.go      | 70 ++++++++++++++++++++++++++++++-
 internal/extract/schema/jsonld.go | 17 +++++++-
 2 files changed, 83 insertions(+), 4 deletions(-)

diff --git a/internal/extract/pipeline.go b/internal/extract/pipeline.go
index 1a079d3..c3d1d74 100644
--- a/internal/extract/pipeline.go
+++ b/internal/extract/pipeline.go
@@ -1,9 +1,12 @@
 package extract
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -13,7 +16,8 @@ import (
 )
 
 type Pipeline struct {
-	client *http.Client
+	client         *http.Client
+	flareSolverURL string
 }
 
 func NewPipeline() *Pipeline {
@@ -21,6 +25,16 @@ func NewPipeline() *Pipeline {
 		client: &http.Client{
 			Timeout: 15 * time.Second,
 		},
+		flareSolverURL: func() string {
+		v := os.Getenv("FLARESOLVERR_URL")
+		if v == "0" || v == "" {
+			return ""
+		}
+		if v == "1" {
+			return "http://localhost:8191/v1"
+		}
+		return v
+	}(),
 	}
 }
 
@@ -32,7 +46,15 @@ type Result struct {
 func (p *Pipeline) Extract(targetURL string) *Result {
 	body, err := p.fetch(targetURL)
 	if err != nil {
-		return &Result{Error: fmt.Errorf("fetching page: %w", err)}
+		if p.flareSolverURL != "" {
+			flareBody, flareErr := p.fetchViaFlareSolver(targetURL)
+			if flareErr != nil {
+				return &Result{Error: fmt.Errorf("fetching page: %w (flaresolverr: %v)", err, flareErr)}
+			}
+			body = flareBody
+		} else {
+			return &Result{Error: fmt.Errorf("fetching page: %w", err)}
+		}
 	}
 
 	if recipe, ok := schema.Extract(body); ok {
@@ -83,6 +105,50 @@ func (p *Pipeline) fetch(url string) (string, error) {
 	return string(body), nil
 }
 
+func (p *Pipeline) fetchViaFlareSolver(targetURL string) (string, error) {
+	reqBody, _ := json.Marshal(map[string]any{
+		"cmd":        "request.get",
+		"url":        targetURL,
+		"maxTimeout": 60000,
+	})
+
+	req, err := http.NewRequest("POST", p.flareSolverURL, bytes.NewReader(reqBody))
+	if err != nil {
+		return "", fmt.Errorf("creating flaresolverr request: %w", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{Timeout: 90 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("flaresolverr request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("reading flaresolverr response: %w", err)
+	}
+
+	var result struct {
+		Status   string `json:"status"`
+		Message  string `json:"message"`
+		Solution struct {
+			Response string `json:"response"`
+			Status   int    `json:"status"`
+		} `json:"solution"`
+	}
+	if err := json.Unmarshal(body, &result); err != nil {
+		return "", fmt.Errorf("parsing flaresolverr response: %w", err)
+	}
+
+	if result.Status != "ok" {
+		return "", fmt.Errorf("flaresolverr: %s", result.Message)
+	}
+
+	return result.Solution.Response, nil
+}
+
 func domainOf(url string) string {
 	parts := strings.SplitAfter(url, "://")
 	if len(parts) < 2 {
diff --git a/internal/extract/schema/jsonld.go b/internal/extract/schema/jsonld.go
index 9887797..22b898e 100644
--- a/internal/extract/schema/jsonld.go
+++ b/internal/extract/schema/jsonld.go
@@ -137,12 +137,25 @@ func parseJSONLD(content string) *models.Recipe {
 	return recipe
 }
 
+func isRecipeType(typ interface{}) bool {
+	switch v := typ.(type) {
+	case string:
+		return v == "Recipe"
+	case []interface{}:
+		for _, item := range v {
+			if s, ok := item.(string); ok && s == "Recipe" {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 func findRecipeObject(v interface{}) interface{} {
 	switch val := v.(type) {
 	case map[string]interface{}:
 		if typ, ok := val["@type"]; ok {
-			typeStr := fmt.Sprintf("%v", typ)
-			if typeStr == "Recipe" {
+			if isRecipeType(typ) {
 				return val
 			}
 		}
-- 
2.51.2