From 8f0878c7a4df4f10f49effc2feebbba28a04a5e1 Mon Sep 17 00:00:00 2001 From: dhawos Date: Sat, 27 Jun 2026 20:33:53 +0200 Subject: [PATCH] working pagination --- feeds/lille.go | 62 +++++++++++++++++++++++++++++++++++++++----- feeds/types.go | 9 +++++-- server/controller.go | 15 +---------- 3 files changed, 63 insertions(+), 23 deletions(-) diff --git a/feeds/lille.go b/feeds/lille.go index 562be73..e190649 100644 --- a/feeds/lille.go +++ b/feeds/lille.go @@ -5,9 +5,13 @@ import ( _ "embed" "encoding/json" "fmt" + "strconv" + "strings" + "time" "github.com/elastic/go-elasticsearch/v9" "github.com/elastic/go-elasticsearch/v9/typedapi/esdsl" + "github.com/elastic/go-elasticsearch/v9/typedapi/types" "github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/fieldvaluefactormodifier" "github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/functionboostmode" "github.com/elastic/go-elasticsearch/v9/typedapi/types/enums/functionscoremode" @@ -26,7 +30,38 @@ func LilleFeed(es *elasticsearch.TypedClient) *Feed { DisplayName: "Lille", Description: "Ce fil est en développement. Il est susceptible de changer. Un fil non officiel contenant les posts parlant de Lille et de la MEL.\n Je n'ai aucun lien avec la MEL ou la ville de Lille.", SelectionFunc: func(ctx context.Context, req *FeedRequest) (FeedSkeletonResponse, error) { - decayFunction := esdsl.NewDateDecayFunction().AddDecayFunctionBaseDateMathDuration("createdAt", esdsl.NewDecayPlacementDateMathDuration().Decay(0.5).Scale(esdsl.NewDuration().String("1d"))) + var pitID string + var previousPageScore float64 + var cursorTime time.Time + var previouslastCID string + // If cursor == 0, we must open a point in time to get a consistent view of the data + if req.Cursor == "" { + resp, err := es.OpenPointInTime(indexer.IndexName).KeepAlive("5m").Do(ctx) + if err != nil { + return FeedSkeletonResponse{}, fmt.Errorf("unable to open point in time: %w", err) + } + pitID = resp.Id + cursorTime = time.Now().Truncate(time.Second) + } else { + splitCursor := strings.Split(req.Cursor, "::") + if len(splitCursor) != 4 { + return FeedSkeletonResponse{}, ErrInvalidCursor + } + pitID = splitCursor[0] + var err error + previousPageScore, err = strconv.ParseFloat(splitCursor[1], 64) + if err != nil { + return FeedSkeletonResponse{}, ErrInvalidCursor + } + timestamp, err := strconv.ParseInt(splitCursor[2], 10, 64) + if err != nil { + return FeedSkeletonResponse{}, ErrInvalidCursor + } + previouslastCID = splitCursor[3] + cursorTime = time.Unix(timestamp, 0) + } + + decayFunction := esdsl.NewDateDecayFunction().AddDecayFunctionBaseDateMathDuration("createdAt", esdsl.NewDecayPlacementDateMathDuration().Decay(0.5).Origin(strconv.FormatInt(cursorTime.UnixMilli(), 10)).Scale(esdsl.NewDuration().String("1d"))) boolQuery := esdsl.NewBoolQuery(). Should(esdsl.NewMatchQuery(indexer.TextFieldName, "Lille"), esdsl.NewMatchPhraseQuery(indexer.TextFieldName, "Villeneuve d'Ascq"), @@ -55,16 +90,25 @@ func LilleFeed(es *elasticsearch.TypedClient) *Feed { scoreQuery := esdsl.NewFunctionScoreQuery().Query(popularityQuery).Functions( esdsl.NewFunctionScore().Gauss(decayFunction), ) - res, err := es.Search(). - Index(indexer.IndexName). - Query(scoreQuery). + search := es.Search(). + Pit(esdsl.NewPointInTimeReference().Id(pitID)). Size(int(req.Limit)). - Sort(esdsl.NewSortCombinations().Field("_score")). + Sort(esdsl.NewSortCombinations().Field("_score"), esdsl.NewSortCombinations().Field("cid")) + + if previousPageScore != 0 { + search.SearchAfter( + esdsl.NewFieldValue().Float64(types.Float64(previousPageScore)), + esdsl.NewFieldValue().String(previouslastCID), + ) + } + res, err := search.Query(scoreQuery). Do(ctx) if err != nil { return FeedSkeletonResponse{}, fmt.Errorf("requesting elastic for lille related posts: %w", err) } result := make([]FeedResponse, 0, len(res.Hits.Hits)) + var lastScore float64 + var lastCID string for _, hit := range res.Hits.Hits { var post indexer.PostDocument err := json.Unmarshal(hit.Source_, &post) @@ -72,9 +116,13 @@ func LilleFeed(es *elasticsearch.TypedClient) *Feed { if err != nil { return FeedSkeletonResponse{}, fmt.Errorf("unmarshalling result from elasticsearch: %w", err) } - result = append(result, FeedResponse{Post: post.Cid}) + postResponse := FeedResponse{Post: post.Cid} + result = append(result, postResponse) + lastScore = float64(*hit.Score_) + lastCID = post.Cid } - return FeedSkeletonResponse{Cursor: "", Feed: result}, nil + cursor := fmt.Sprintf("%s::%f::%d::%s", *res.PitId, lastScore, cursorTime.Unix(), lastCID) + return FeedSkeletonResponse{Cursor: cursor, Feed: result}, nil }, } } diff --git a/feeds/types.go b/feeds/types.go index 138830b..b222cd2 100644 --- a/feeds/types.go +++ b/feeds/types.go @@ -1,6 +1,11 @@ package feeds -import "context" +import ( + "context" + "errors" +) + +var ErrInvalidCursor = errors.New("invalid cursor format") type Feed struct { Avatar []byte @@ -12,7 +17,7 @@ type Feed struct { type FeedRequest struct { Lang string - Cursor int64 + Cursor string Limit int32 } diff --git a/server/controller.go b/server/controller.go index e7000de..be90396 100644 --- a/server/controller.go +++ b/server/controller.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "log/slog" - "math" "net/http" "strconv" @@ -92,18 +91,6 @@ func (bfg *BlueskyFeedGenerator) appBskyFeedGetFeedSkeletonHandler(w http.Respon return } cursor := req.URL.Query().Get("cursor") - if cursor == "" { - cursor = "-1" - } - cursorInt64, err := strconv.ParseInt(cursor, 10, 64) - if cursorInt64 < 0 { - cursorInt64 = math.MaxInt64 - } - if err != nil { - slog.Error("could not parse cursor", "cursor", cursor, "err", err) - http.Error(w, "", http.StatusBadRequest) - return - } limit := req.URL.Query().Get("limit") if limit == "" { limit = defaultFeedLimit @@ -139,7 +126,7 @@ func (bfg *BlueskyFeedGenerator) appBskyFeedGetFeedSkeletonHandler(w http.Respon } feedRequest := feeds.FeedRequest{ Lang: "fr", - Cursor: cursorInt64, + Cursor: cursor, Limit: limitInt32, } response, err := algo.SelectionFunc(req.Context(), &feedRequest) -- 2.51.2