(READ ONLY) Margin is an open annotation layer for the internet. Powered by the AT Protocol.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272package api
import ( "context" "encoding/json" "fmt" "net/http" "strings" "sync" "time"
"margin.at/internal/db" "margin.at/internal/logger" "margin.at/internal/xrpc")
var ( failedCardsMu sync.RWMutex failedCards = make(map[string]time.Time))
func isRecentlyFailed(uri string) bool { failedCardsMu.RLock() t, ok := failedCards[uri] failedCardsMu.RUnlock() return ok && time.Since(t) < 30*time.Minute}
func markFailed(uri string) { failedCardsMu.Lock() failedCards[uri] = time.Now() failedCardsMu.Unlock()}
func init() { go func() { for { time.Sleep(10 * time.Minute) failedCardsMu.Lock() for uri, t := range failedCards { if time.Since(t) > 30*time.Minute { delete(failedCards, uri) } } failedCardsMu.Unlock() } }()}
func ensureSembleCardsIndexed(ctx context.Context, database *db.DB, uris []string) { if len(uris) == 0 || database == nil { return }
uniq := make(map[string]struct{}, len(uris)) deduped := make([]string, 0, len(uris)) for _, u := range uris { if u == "" { continue } if _, ok := uniq[u]; ok { continue } uniq[u] = struct{}{} deduped = append(deduped, u) } if len(deduped) == 0 { return }
existingAnnos, _ := database.GetAnnotationsByURIs(deduped) existingBooks, _ := database.GetBookmarksByURIs(deduped)
foundSet := make(map[string]bool, len(existingAnnos)+len(existingBooks)) for _, a := range existingAnnos { foundSet[a.URI] = true } for _, b := range existingBooks { foundSet[b.URI] = true }
missing := make([]string, 0) for _, u := range deduped { if !foundSet[u] && !isRecentlyFailed(u) { missing = append(missing, u) } } if len(missing) == 0 { return }
logger.Info("Active Cache: Fetching %d missing Semble cards...", len(missing)) fetchAndIndexSembleCards(ctx, database, missing)}
func fetchAndIndexSembleCards(ctx context.Context, database *db.DB, uris []string) { sem := make(chan struct{}, 5) var wg sync.WaitGroup
for _, uri := range uris { select { case <-ctx.Done(): return default: }
wg.Add(1) go func(u string) { defer wg.Done()
select { case sem <- struct{}{}: defer func() { <-sem }() case <-ctx.Done(): return }
if err := fetchSembleCard(ctx, database, u); err != nil { markFailed(u) if ctx.Err() == nil { logger.Error("Failed to lazy fetch card %s: %v", u, err) } } }(uri) }
done := make(chan struct{}) go func() { wg.Wait() close(done) }()
select { case <-done: case <-ctx.Done(): return }}
func fetchSembleCard(ctx context.Context, database *db.DB, uri string) error { if database == nil { return fmt.Errorf("nil database") }
if !strings.HasPrefix(uri, "at://") { return fmt.Errorf("invalid uri") } uriWithoutScheme := strings.TrimPrefix(uri, "at://") parts := strings.Split(uriWithoutScheme, "/") if len(parts) < 3 { return fmt.Errorf("invalid uri parts: expected at least 3 parts") } did, _, _ := parts[0], parts[1], parts[2]
record, err := xrpc.SlingshotClient.GetRecord(ctx, uri) if err != nil { return fetchSembleCardFromPDS(ctx, database, uri, did, parts[1], parts[2]) }
var card xrpc.SembleCard if err := json.Unmarshal(record.Value, &card); err != nil { return err }
return indexSembleCard(database, uri, did, &card)}
func fetchSembleCardFromPDS(ctx context.Context, database *db.DB, uri, did, collection, rkey string) error { pds, err := xrpc.ResolveDIDToPDS(did) if err != nil { return fmt.Errorf("failed to resolve PDS: %w", err) }
fetchURL := fmt.Sprintf("%s/xrpc/com.atproto.repo.getRecord?repo=%s&collection=%s&rkey=%s", pds, did, collection, rkey)
req, err := http.NewRequestWithContext(ctx, "GET", fetchURL, nil) if err != nil { return err }
client := &http.Client{Timeout: 5 * time.Second} resp, err := client.Do(req) if err != nil { return fmt.Errorf("failed to fetch record: %w", err) } defer resp.Body.Close()
if resp.StatusCode != 200 { return fmt.Errorf("unexpected status %d", resp.StatusCode) }
var output xrpc.GetRecordOutput if err := json.NewDecoder(resp.Body).Decode(&output); err != nil { return err }
var card xrpc.SembleCard if err := json.Unmarshal(output.Value, &card); err != nil { return err }
return indexSembleCard(database, uri, did, &card)}
func indexSembleCard(database *db.DB, uri, did string, card *xrpc.SembleCard) error { createdAt := card.GetCreatedAtTime() content, err := card.ParseContent() if err != nil { return err }
switch card.Type { case "NOTE": note, ok := content.(*xrpc.SembleNoteContent) if !ok { return fmt.Errorf("invalid note content") }
targetSource := card.URL if targetSource == "" { return fmt.Errorf("missing target source") }
targetHash := db.HashURL(targetSource) motivation := "commenting" bodyValue := note.Text
annotation := &db.Annotation{ URI: uri, AuthorDID: did, Motivation: motivation, BodyValue: &bodyValue, TargetSource: targetSource, TargetHash: targetHash, CreatedAt: createdAt, IndexedAt: time.Now(), } return database.CreateAnnotation(annotation)
case "URL": urlContent, ok := content.(*xrpc.SembleURLContent) if !ok { return fmt.Errorf("invalid url content") }
source := urlContent.URL if source == "" { return fmt.Errorf("missing source") } sourceHash := db.HashURL(source)
var titlePtr *string if urlContent.Metadata != nil && urlContent.Metadata.Title != "" { t := urlContent.Metadata.Title titlePtr = &t }
bookmark := &db.Bookmark{ URI: uri, AuthorDID: did, Source: source, SourceHash: sourceHash, Title: titlePtr, CreatedAt: createdAt, IndexedAt: time.Now(), } return database.CreateBookmark(bookmark) }
return nil}