diff --git a/chrome-extension/src/description_generation.js b/chrome-extension/src/description_generation.js
index 82955b1..9f0e033 100644
--- a/chrome-extension/src/description_generation.js
+++ b/chrome-extension/src/description_generation.js
@@ -59,7 +59,6 @@ Document: {{content}}`;
document.getElementById("descLoading").style.display = "none";
console.error("Error generating description with Ollama:", error);
- updateStatus(`Error generating description: ${error.message}`);
return [];
}
}
diff --git a/cmd/root.go b/cmd/root.go
index 3d91212..bde2207 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -76,11 +76,20 @@ func (m rootAppModel) Init() tea.Cmd { return nil }
func (m rootAppModel) updateTable() rootAppModel {
bmCount := m.rowsCount
-
- bookmarks, err := store.SearchBookmarks(m.db, m.input.Value())
- if err != nil {
- log.Panicln(err)
- return m
+ var bookmarks []store.Bookmark
+ var err error
+ if m.input.Value() != "" {
+ bookmarks, err = store.SearchBookmarks(m.db, m.input.Value())
+ if err != nil {
+ log.Panicln(err)
+ return m
+ }
+ } else {
+ bookmarks, err = store.GetBookmarks(m.db)
+ if err != nil {
+ log.Panicln(err)
+ return m
+ }
}
if len(bookmarks) == bmCount {
@@ -269,6 +278,7 @@ file sync service. This is sort-of explained the following blog post:
input.Placeholder = "Search / Filter"
m := rootAppModel{db: db, table: t, input: input, currentIndex: 1, rowsCount: 0, mode: NORMAL}
+ m = m.updateTable()
prog := tea.NewProgram(m, tea.WithAltScreen())
diff --git a/cmd/sql.go b/cmd/sql.go
index e90bc27..1618429 100644
--- a/cmd/sql.go
+++ b/cmd/sql.go
@@ -14,6 +14,7 @@ import (
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/lipgloss/table"
"github.com/charmbracelet/x/term"
+ "github.com/lukasmwerner/mark/search"
"github.com/lukasmwerner/mark/store"
"github.com/spf13/cobra"
)
@@ -92,30 +93,25 @@ var sqlCmd = &cobra.Command{
query = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
} else if input == ".schema" {
query = "SELECT sql FROM sqlite_master WHERE type='table' ORDER BY name;"
- } else if search, ok := strings.CutPrefix(input, ".web-search"); ok {
- results, err := WebSearchBookmarks(db, search, ftsRank)
+ } else if s, ok := strings.CutPrefix(input, ".web-search"); ok {
+ results, err := search.Search(db, s)
if err != nil {
fmt.Println("err: " + err.Error())
continue
}
- renderBookmarks(results)
+ renderSearchResults(results)
query = ""
continue
- } else if search, ok := strings.CutPrefix(input, ".semantic "); ok {
- fmt.Printf("semantic: '%s'\n", search)
- rows, err := db.Query(`SELECT b.url, b.title, b.description, b.tags, distance
- FROM bookmark_embeddings e
- JOIN Bookmarks b ON e.document_id = b.id
- WHERE e.embedding MATCH embed('embeddinggemma', concat_ws(' ', 'task: search result | query: ', ?)) and k = 100 and distance <= 1.21
- ORDER BY distance;`, search)
+ } else if s, ok := strings.CutPrefix(input, ".semantic "); ok {
+ fmt.Printf("semantic: '%s'\n", s)
+ results, err := search.Semantic(db, s)
if err != nil {
fmt.Println("err: " + err.Error())
continue
}
- renderResults(rows)
+ renderBookmarks(results)
query = ""
- rows.Close()
continue
} else if search, ok := strings.CutPrefix(input, ".search "); ok {
fmt.Printf("search: '%s'\n", search)
@@ -162,6 +158,15 @@ func init() {
// sqlCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
+func renderSearchResults(results []search.Result) {
+ bookmarks := make([]store.Bookmark, len(results))
+ for i, r := range results {
+ bookmarks[i] = r.Bookmark
+ }
+ renderBookmarks(bookmarks)
+
+}
+
func renderBookmarks(bookmarks []store.Bookmark) {
if bookmarks == nil {
return
diff --git a/cmd/web-search.go b/cmd/web-search.go
index 64ef62d..4601ccc 100644
--- a/cmd/web-search.go
+++ b/cmd/web-search.go
@@ -7,167 +7,17 @@ import (
"fmt"
"log"
"net/http"
- "sort"
- "strings"
_ "embed"
"github.com/a-h/templ"
+ "github.com/lukasmwerner/mark/search"
"github.com/lukasmwerner/mark/store"
"github.com/lukasmwerner/mark/web"
"github.com/lukasmwerner/mark/web/static"
"github.com/spf13/cobra"
)
-type RankingMethod string
-
-const (
- recency RankingMethod = "rowid DESC, rank DESC" // rank based on rowid will be an issue when going across users
- ftsRank RankingMethod = "rank DESC, rowid DESC"
-)
-
-func WebSearchBookmarks(db *store.DB, query string, ranker RankingMethod) ([]store.Bookmark, error) {
- bookmarks := []store.Bookmark{}
-
- partials := ""
- for _, field := range strings.Fields(query) {
- partial := ""
- switch field {
- case "NOT":
- partial = field
- case "OR":
- partial = field
- case "AND":
- partial = field
- default:
- partial = field + "*"
-
- }
- partials += " " + partial
- }
-
- rows, err := db.Query(
- `
- SELECT
- url,
- title,
- description,
- rowid,
- tags
- FROM (
- SELECT
- url,
- title,
- description,
- tags,
- rowid,
- rank,
- 1 AS priority
- FROM Bookmarks_fts
- WHERE Bookmarks_fts MATCH ?
-
- UNION
-
- SELECT
- url,
- title,
- description,
- tags,
- rowid,
- rank,
- 2 AS priority
- FROM Bookmarks_fts
- WHERE Bookmarks_fts MATCH ?)
-
- GROUP BY url
- ORDER BY MIN(priority) ASC, `+string(ranker)+`;`,
- query, partials)
-
- // rows, err := db.Query(
- // `SELECT url, title, description, tags FROM Bookmarks_fts WHERE Bookmarks_fts MATCH ? ORDER BY rowid DESC, rank DESC;`,
- // query)
- if err != nil {
- return bookmarks, err
- }
- defer rows.Close()
-
- for rows.Next() {
- var b store.Bookmark
- var tags string
- var id int
- err := rows.Scan(&b.Url, &b.Title, &b.Description, &id, &tags)
- if err != nil {
- return bookmarks, err
- }
- b.Tags = strings.Split(tags, ", ")
- bookmarks = append(bookmarks, b)
- }
-
- return bookmarks, nil
-}
-
-func mergeResults(results ...[]store.Bookmark) []store.Bookmark {
- type rankedBookmark struct {
- bookmark store.Bookmark
- score float64
- bestRank int
- firstSet int
- }
-
- merged := map[string]*rankedBookmark{}
- for setIndex, resultSet := range results {
- for rank, bm := range resultSet {
- key := bm.Url
- if key == "" {
- key = bm.Title
- }
-
- // Reciprocal rank fusion: bookmarks that rank highly in one or more
- // result sets bubble toward the front of the merged list.
- score := 1.0 / float64(rank+1)
- if existing, ok := merged[key]; ok {
- existing.score += score
- if rank < existing.bestRank {
- existing.bestRank = rank
- }
- continue
- }
-
- merged[key] = &rankedBookmark{
- bookmark: bm,
- score: score,
- bestRank: rank,
- firstSet: setIndex,
- }
- }
- }
-
- outputResults := make([]rankedBookmark, 0, len(merged))
- for _, bm := range merged {
- outputResults = append(outputResults, *bm)
- }
-
- sort.SliceStable(outputResults, func(i, j int) bool {
- if outputResults[i].score != outputResults[j].score {
- return outputResults[i].score > outputResults[j].score
- }
- if outputResults[i].bestRank != outputResults[j].bestRank {
- return outputResults[i].bestRank < outputResults[j].bestRank
- }
- if outputResults[i].firstSet != outputResults[j].firstSet {
- return outputResults[i].firstSet < outputResults[j].firstSet
- }
- return outputResults[i].bookmark.Url < outputResults[j].bookmark.Url
- })
-
- bookmarks := make([]store.Bookmark, len(outputResults))
- for i, bm := range outputResults {
- bookmarks[i] = bm.bookmark
- }
-
- return bookmarks
-}
-
var searchCmd = &cobra.Command{
Use: "search",
Short: `[EXPERIMENTAL] google search like interface`,
@@ -188,21 +38,13 @@ var searchCmd = &cobra.Command{
http.Redirect(w, r, "/", http.StatusTemporaryRedirect)
return
}
- fts_results, err := WebSearchBookmarks(db, q, ftsRank)
- if err != nil {
- w.WriteHeader(http.StatusInternalServerError)
- fmt.Fprintln(w, "oops we had something go wrong.")
- fmt.Fprintln(w, err.Error())
- return
- }
- semantic_results, err := store.SemanticSearchBookmarks(db, q)
+ results, err := search.Search(db, q)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintln(w, "oops we had something go wrong.")
fmt.Fprintln(w, err.Error())
return
}
- results := mergeResults(fts_results, semantic_results)
templ.Handler(web.ResultsPage("lukaswerner.com", q, "(FTS + Embeddings) RRF", results)).ServeHTTP(w, r)
})
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
diff --git a/go.mod b/go.mod
index d7c78a1..d479993 100644
--- a/go.mod
+++ b/go.mod
@@ -4,7 +4,7 @@ go 1.24.1
require (
github.com/PuerkitoBio/goquery v1.9.2
- github.com/a-h/templ v0.3.960
+ github.com/a-h/templ v0.3.1001
github.com/charmbracelet/bubbles v0.21.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/huh v0.5.2
diff --git a/go.sum b/go.sum
index 2c42498..cc5be1d 100644
--- a/go.sum
+++ b/go.sum
@@ -2,6 +2,8 @@ github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/a-h/templ v0.3.960 h1:trshEpGa8clF5cdI39iY4ZrZG8Z/QixyzEyUnA7feTM=
github.com/a-h/templ v0.3.960/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
+github.com/a-h/templ v0.3.1001 h1:yHDTgexACdJttyiyamcTHXr2QkIeVF1MukLy44EAhMY=
+github.com/a-h/templ v0.3.1001/go.mod h1:oCZcnKRf5jjsGpf2yELzQfodLphd2mwecwG4Crk5HBo=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/asg017/sqlite-vec-go-bindings v0.1.6 h1:Nx0jAzyS38XpkKznJ9xQjFXz2X9tI7KqjwVxV8RNoww=
diff --git a/search/data.go b/search/data.go
new file mode 100644
index 0000000..d9c9051
--- /dev/null
+++ b/search/data.go
@@ -0,0 +1,9 @@
+package search
+
+import "github.com/lukasmwerner/mark/store"
+
+type Result struct {
+ store.Bookmark
+ Source Source
+ Rank float64
+}
diff --git a/search/fts.go b/search/fts.go
new file mode 100644
index 0000000..4c964a0
--- /dev/null
+++ b/search/fts.go
@@ -0,0 +1,84 @@
+package search
+
+import (
+ "strings"
+
+ "github.com/lukasmwerner/mark/store"
+)
+
+func FullText5(db *store.DB, query string) ([]store.Bookmark, error) {
+ bookmarks := []store.Bookmark{}
+
+ rows, err := db.Query(`SELECT url, title, description, tags
+ FROM Bookmarks_fts
+ WHERE Bookmarks_fts MATCH ?
+ ORDER BY bm25(Bookmarks_fts) DESC;`, query)
+ if err != nil {
+ return bookmarks, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var b store.Bookmark
+ var tags string
+ err := rows.Scan(&b.Url, &b.Title, &b.Description, &tags)
+ if err != nil {
+ return bookmarks, err
+ }
+ b.Tags = strings.Split(tags, ", ")
+ bookmarks = append(bookmarks, b)
+ }
+
+ return bookmarks, nil
+}
+
+func PartialText(db *store.DB, query string) ([]store.Bookmark, error) {
+ bookmarks := []store.Bookmark{}
+
+ fuzzy_query := ""
+ for _, field := range strings.Fields(query) {
+ fragment := ""
+ switch field {
+ case "NOT":
+ fragment = field
+ case "OR":
+ fragment = field
+ case "AND":
+ fragment = field
+ default:
+ fragment = field + "*"
+
+ }
+ fuzzy_query += " " + fragment
+ }
+
+ rows, err := db.Query(`SELECT
+ url,
+ title,
+ description,
+ rowid,
+ tags
+ FROM Bookmarks_fts
+ WHERE Bookmarks_fts MATCH ?
+ ORDER BY bm25(Bookmarks_fts) DESC;`,
+ fuzzy_query)
+
+ if err != nil {
+ return bookmarks, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var b store.Bookmark
+ var tags string
+ var id int
+ err := rows.Scan(&b.Url, &b.Title, &b.Description, &id, &tags)
+ if err != nil {
+ return bookmarks, err
+ }
+ b.Tags = strings.Split(tags, ", ")
+ bookmarks = append(bookmarks, b)
+ }
+
+ return bookmarks, nil
+}
diff --git a/search/merge.go b/search/merge.go
new file mode 100644
index 0000000..6983b0c
--- /dev/null
+++ b/search/merge.go
@@ -0,0 +1,83 @@
+package search
+
+import (
+ "sort"
+
+ "github.com/lukasmwerner/mark/store"
+)
+
+type Source string
+
+const (
+ FTS = Source("FTS")
+ PartialFTS = Source("Partial FTS")
+ Embedding = Source("Embedding")
+)
+
+func MergeResults(sources []Source, results ...[]store.Bookmark) []Result {
+ type rankedBookmark struct {
+ bookmark store.Bookmark
+ score float64
+ bestRank int
+ firstSet int
+ source Source
+ }
+
+ merged := map[string]*rankedBookmark{}
+ for setIndex, resultSet := range results {
+ for rank, bm := range resultSet {
+ key := bm.Url
+ if key == "" {
+ key = bm.Title
+ }
+
+ // Reciprocal rank fusion: bookmarks that rank highly in one or more
+ // result sets bubble toward the front of the merged list.
+ score := 1.0 / float64(rank+1)
+ if existing, ok := merged[key]; ok {
+ existing.score += score
+ if rank < existing.bestRank {
+ existing.bestRank = rank
+ }
+ continue
+ }
+
+ merged[key] = &rankedBookmark{
+ bookmark: bm,
+ score: score,
+ bestRank: rank,
+ firstSet: setIndex,
+ source: sources[setIndex],
+ }
+ }
+ }
+
+ outputResults := make([]rankedBookmark, 0, len(merged))
+ for _, bm := range merged {
+ outputResults = append(outputResults, *bm)
+ }
+
+ sort.SliceStable(outputResults, func(i, j int) bool {
+ if outputResults[i].score != outputResults[j].score {
+ return outputResults[i].score > outputResults[j].score
+ }
+ if outputResults[i].bestRank != outputResults[j].bestRank {
+ return outputResults[i].bestRank < outputResults[j].bestRank
+ }
+ if outputResults[i].firstSet != outputResults[j].firstSet {
+ return outputResults[i].firstSet < outputResults[j].firstSet
+ }
+ return outputResults[i].bookmark.Url < outputResults[j].bookmark.Url
+ })
+
+ bookmarks := make([]Result, len(outputResults))
+ for i, bm := range outputResults {
+ bookmarks[i] = Result{
+ Bookmark: bm.bookmark,
+ Source: bm.source,
+ Rank: bm.score,
+ }
+ }
+
+ return bookmarks
+}
diff --git a/search/search.go b/search/search.go
new file mode 100644
index 0000000..4d9e4fe
--- /dev/null
+++ b/search/search.go
@@ -0,0 +1,24 @@
+package search
+
+import "github.com/lukasmwerner/mark/store"
+
+func Search(db *store.DB, query string) ([]Result, error) {
+ results := []Result{}
+ // TODO: make these searches concurrent?
+ fts5, err := FullText5(db, query)
+ if err != nil {
+ return results, err
+ }
+ partials, err := PartialText(db, query)
+ if err != nil {
+ return results, err
+ }
+ semantic, err := Semantic(db, query)
+ if err != nil {
+ return results, err
+ }
+
+ results = MergeResults([]Source{FTS, Embedding, PartialFTS}, fts5, semantic, partials)
+
+ return results, nil
+}
diff --git a/search/semantic.go b/search/semantic.go
new file mode 100644
index 0000000..499d4f3
--- /dev/null
+++ b/search/semantic.go
@@ -0,0 +1,33 @@
+package search
+
+import (
+ "strings"
+
+ "github.com/lukasmwerner/mark/store"
+)
+
+func Semantic(db *store.DB, query string) ([]store.Bookmark, error) {
+ bookmarks := []store.Bookmark{}
+ rows, err := db.Query(`SELECT b.url, b.title, b.description, b.tags
+ FROM bookmark_embeddings e
+ JOIN Bookmarks b ON e.document_id = b.id
+ WHERE e.embedding MATCH embed('embeddinggemma', concat_ws(' ', 'task: search result | query: ', ?)) and k = 100 and distance <= 1.21
+ ORDER BY distance;`, query)
+ if err != nil {
+ return bookmarks, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var b store.Bookmark
+ var tags string
+ err := rows.Scan(&b.Url, &b.Title, &b.Description, &tags)
+ if err != nil {
+ return bookmarks, err
+ }
+ b.Tags = strings.Split(tags, ", ")
+ bookmarks = append(bookmarks, b)
+ }
+
+ return bookmarks, nil
+}
diff --git a/store/database.go b/store/database.go
index 18bc6bd..54161ac 100644
--- a/store/database.go
+++ b/store/database.go
@@ -304,6 +304,32 @@ func InsertBookmark(db *DB, bookmark Bookmark) (BookmarkId, error) {
return BookmarkId(id), err
}
+func SearchBookmarks(db *DB, query string) ([]Bookmark, error) {
+ bookmarks := []Bookmark{}
+
+ rows, err := db.Query(`SELECT url, title, description, tags
+ FROM Bookmarks_fts
+ WHERE Bookmarks_fts MATCH ?
+ ORDER BY bm25(Bookmarks_fts) DESC;`, query)
+ if err != nil {
+ return bookmarks, err
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var b Bookmark
+ var tags string
+ err := rows.Scan(&b.Url, &b.Title, &b.Description, &tags)
+ if err != nil {
+ return bookmarks, err
+ }
+ b.Tags = strings.Split(tags, ", ")
+ bookmarks = append(bookmarks, b)
+ }
+
+ return bookmarks, nil
+}
+
func GetBookmark(db *DB, query_url string) (Bookmark, error) {
var b Bookmark
var tags string
@@ -354,39 +380,12 @@ func GetBookmark(db *DB, query_url string) (Bookmark, error) {
return b, nil
}
-func SemanticSearchBookmarks(db *DB, query string) ([]Bookmark, error) {
- bookmarks := []Bookmark{}
- rows, err := db.Query(`SELECT b.url, b.title, b.description, b.tags
- FROM bookmark_embeddings e
- JOIN Bookmarks b ON e.document_id = b.id
- WHERE e.embedding MATCH embed('embeddinggemma', concat_ws(' ', 'task: search result | query: ', ?)) and k = 100 and distance <= 1.21
- ORDER BY distance;`, query)
- if err != nil {
- return bookmarks, err
- }
- defer rows.Close()
-
- for rows.Next() {
- var b Bookmark
- var tags string
- err := rows.Scan(&b.Url, &b.Title, &b.Description, &tags)
- if err != nil {
- return bookmarks, err
- }
- b.Tags = strings.Split(tags, ", ")
- bookmarks = append(bookmarks, b)
- }
+func GetBookmarks(db *DB) ([]Bookmark, error) {
- return bookmarks, nil
-}
-
-func SearchBookmarks(db *DB, query string) ([]Bookmark, error) {
bookmarks := []Bookmark{}
- query = strings.Join(strings.Fields(query), "* ") + "*"
- rows, err := db.Query(`SELECT url, title, description, tags
- FROM Bookmarks_fts
- WHERE Bookmarks_fts MATCH ?
- ORDER BY bm25(Bookmarks_fts) DESC;`, query)
+ rows, err := db.Query(`SELECT url, title, description, tags
+ FROM Bookmarks
+ ORDER BY rowid DESC;`)
if err != nil {
return bookmarks, err
}
diff --git a/web/results.templ b/web/results.templ
index c51f15a..85c9117 100644
--- a/web/results.templ
+++ b/web/results.templ
@@ -1,7 +1,7 @@
package web
import (
- "github.com/lukasmwerner/mark/store"
+ "github.com/lukasmwerner/mark/search"
"net/url"
)
@@ -14,7 +14,7 @@ func DomainOnly(u string) string {
}
-templ ResultsPage(markHostname string, query string, orderBy string, results []store.Bookmark) {
+templ ResultsPage(markHostname string, query string, orderBy string, results []search.Result) {
@Page() {
@Head(query + " - mark: " + markHostname)
@Body() {
@@ -52,6 +52,7 @@ templ ResultsPage(markHostname string, query string, orderBy string, results []s
{ result.Title }
{ DomainOnly(result.Url) }
+ { result.Source }
{ result.Description }
}
diff --git a/web/static/style.css b/web/static/style.css
index 0d6dd6e..1732c9a 100644
--- a/web/static/style.css
+++ b/web/static/style.css
@@ -137,6 +137,18 @@ a:visited {
margin-bottom: 0.5em;
display: inline-flex;
}
+ .algorithm {
+ color: #D3C6AA;
+ background: #3D484D;
+ border-radius: 50px;
+ font-size: 0.9rem;
+ padding: 0.5em;
+ padding-left: 1em;
+ padding-right: 1em;
+ margin-top: 0.5em;
+ margin-bottom: 0.5em;
+ display: inline-flex;
+ }
p {
color: #C4C0B9;
margin-top: 0.5em