Mbin link post body text
", + "mediaType": "text/html", + "source": "https://video.troed.se/w/5FqDa2jVqCcQWxetJEivnw", + "url": "https://fedia.io/m/startrek@startrek.website/t/4194502", + "attachment": [ + { + "type": "Link", + "href": "https://video.troed.se/w/5FqDa2jVqCcQWxetJEivnw" + } + ], + "commentsEnabled": true, + "sensitive": false, + "stickied": false, + "published": "2026-09-27T18:04:11+00:00" +} diff --git a/internal/ap/vocab.go b/internal/ap/vocab.go index d3d6b17..ae577a2 100644 --- a/internal/ap/vocab.go +++ b/internal/ap/vocab.go @@ -259,14 +259,26 @@ func (o *Object) UnmarshalJSON(data []byte) error { // survives), absent means a self-delete (it does not). A RawMessage is // empty only when the key was genuinely absent, so presence is decided on // the KEY and the text is read only when there is a JSON string to read. + // + // `source` is shadowed too, and only a JSON object is decoded into Source. + // A string `source` is never markdown (Mbin, for one, sends a link post's + // external URL there); like any other non-object shape (number, array, + // bool, null) it leaves Source nil instead of failing the whole object. A + // malformed object still fails the parse, with an error naming `source`. var wire struct { objectAlias Summary json.RawMessage `json:"summary"` + Source json.RawMessage `json:"source"` } if err := json.Unmarshal(data, &wire); err != nil { return err } *o = Object(wire.objectAlias) + if len(wire.Source) > 0 && wire.Source[0] == '{' { + if err := json.Unmarshal(wire.Source, &o.Source); err != nil { + return fmt.Errorf("object %q source: %w", o.ID, err) + } + } if len(wire.Summary) > 0 { o.summaryPresent = true // Anything that is not a JSON string (null, and any wrong-typed value @@ -610,7 +622,9 @@ func unmarshalOneOrMany[S ~[]E, E any](data []byte, out *S) error { } // Source carries the original markdown of an object (Lemmy always includes -// it alongside the rendered HTML content). +// it alongside the rendered HTML content). Only a JSON object `source` is +// decoded; a string is never markdown and, like any other non-object shape, +// leaves Source nil (see Object.UnmarshalJSON). type Source struct { Content string `json:"content,omitempty"` MediaType string `json:"mediaType,omitempty"` diff --git a/internal/ap/vocab_mbin_test.go b/internal/ap/vocab_mbin_test.go new file mode 100644 index 0000000..5a14b98 --- /dev/null +++ b/internal/ap/vocab_mbin_test.go @@ -0,0 +1,150 @@ +package ap + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Fixture provenance: page_mbin_fedia_io.json is hand-built (no live fetch) +// from the output shape of Mbin's EntryPageFactory for a link entry. Mbin +// sends `source` as the bare external URL string instead of Lemmy's +// {content, mediaType} markdown object; that string must never become the +// object's markdown source. +const ( + mbinPageFixture = "page_mbin_fedia_io.json" + mbinPageID = "https://fedia.io/m/startrek@startrek.website/t/4194502" + mbinAuthorID = "https://fedia.io/u/troed" + mbinGroupID = "https://startrek.website/c/startrek" + mbinVideoURL = "https://video.troed.se/w/5FqDa2jVqCcQWxetJEivnw" +) + +// assertNoMarkdownSource accepts either a nil Source or one with no +// content: which of the two a parser produces is not the contract, only that +// the URL string is never read as markdown. +func assertNoMarkdownSource(t *testing.T, page *Object) { + t.Helper() + if page.Source != nil { + assert.Empty(t, page.Source.Content, + "Mbin's string source is a URL, never markdown source content") + } +} + +func TestParseMbinLinkPageWithStringSource(t *testing.T) { + page, err := ParseObject(loadFixture(t, mbinPageFixture)) + require.NoError(t, err, "an Mbin Page with a string source must parse") + + assert.Equal(t, TypePage, page.Type) + assert.Equal(t, mbinPageID, page.ID) + assert.Equal(t, mbinAuthorID, page.AttributedTo.FirstID()) + assert.Equal(t, "Star Trek clip shared from fedia.io", page.Name) + assert.Equal(t, "Mbin link post body text
", page.Content) + require.Len(t, page.Attach, 1) + assert.Equal(t, "Link", page.Attach[0].Type) + assert.Equal(t, mbinVideoURL, page.Attach[0].Href) + assertNoMarkdownSource(t, page) +} + +func TestParseAnnouncedMbinLinkPageWithStringSource(t *testing.T) { + // The inbox shape: the community announces the Mbin author's Create with + // the Page inline. + announce, err := json.Marshal(map[string]any{ + "type": "Announce", + "id": "https://startrek.website/activities/announce/create/mbin-4194502", + "actor": mbinGroupID, + "to": []any{PublicAudience}, + "cc": []any{mbinGroupID + "/followers"}, + "object": map[string]any{ + "type": "Create", + "id": "https://fedia.io/f/object/7c1b2f4e-3d5a-4b8e-9f10-2a6c8d4e5b71", + "actor": mbinAuthorID, + "to": []any{mbinGroupID, PublicAudience}, + "cc": []any{mbinAuthorID + "/followers"}, + "audience": mbinGroupID, + "object": json.RawMessage(loadFixture(t, mbinPageFixture)), + }, + }) + require.NoError(t, err) + + activity, err := ParseObject(announce) + require.NoError(t, err, "an announced Mbin Page with a string source must parse") + + require.NotNil(t, activity.Object, "Announce carries the Create") + require.NotNil(t, activity.Object.Object, "Create carries the Page") + page := activity.Object.Object + assert.Equal(t, TypePage, page.Type) + assert.Equal(t, mbinPageID, page.ID) + assertNoMarkdownSource(t, page) +} + +func TestMarshalMbinLinkPageNeverEmitsStringSource(t *testing.T) { + page, err := ParseObject(loadFixture(t, mbinPageFixture)) + require.NoError(t, err, "an Mbin Page with a string source must parse") + + out, err := json.Marshal(page) + require.NoError(t, err) + var fields map[string]json.RawMessage + require.NoError(t, json.Unmarshal(out, &fields)) + + // Only the `source` key is inspected: the URL legitimately stays under + // `attachment`. + assert.NotContains(t, fields, "source", + "a string source parses to no Source, so none may be re-emitted") +} + +const sourceShapePageID = "https://lemmy.test/post/1" + +// sourceShapePage builds a minimal Page with sourceJSON spliced in raw as the +// `source` value; an empty sourceJSON omits the key entirely. +func sourceShapePage(sourceJSON string) []byte { + fields := `"type":"Page","id":"` + sourceShapePageID + `","content":"body
"` + if sourceJSON != "" { + fields += `,"source":` + sourceJSON + } + return []byte("{" + fields + "}") +} + +// TestParseObjectSourceShapes pins that `source` is optional in every shape: +// only a JSON object is read as markdown source, and any other shape leaves +// Source nil instead of failing the object (and with it, a whole outbox page). +func TestParseObjectSourceShapes(t *testing.T) { + cases := []struct { + name string + sourceJSON string + want *Source + }{ + {name: "absent", sourceJSON: "", want: nil}, + {name: "null", sourceJSON: `null`, want: nil}, + {name: "empty string", sourceJSON: `""`, want: nil}, + { + name: "markdown object", + sourceJSON: `{"content":"**hi**","mediaType":"text/markdown"}`, + want: &Source{Content: "**hi**", MediaType: "text/markdown"}, + }, + {name: "number", sourceJSON: `42`, want: nil}, + {name: "array", sourceJSON: `["a","b"]`, want: nil}, + {name: "bool", sourceJSON: `true`, want: nil}, + } + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + page, err := ParseObject(sourceShapePage(testCase.sourceJSON)) + require.NoError(t, err, "a Page with source %s must parse", testCase.sourceJSON) + assert.Equal(t, testCase.want, page.Source) + }) + } +} + +// TestParseObjectMalformedSourceObjectErrorNamesSource pins that a source +// object with a wrong-typed field still fails, and that the error names the +// `source` field and the object rather than reading like the top-level +// `content` was wrong. +func TestParseObjectMalformedSourceObjectErrorNamesSource(t *testing.T) { + _, err := ParseObject(sourceShapePage(`{"content":5}`)) + require.Error(t, err) + assert.ErrorContains(t, err, "source") + assert.ErrorContains(t, err, sourceShapePageID) + var typeError *json.UnmarshalTypeError + assert.ErrorAs(t, err, &typeError, "the wrap must keep the decode cause") +} diff --git a/internal/ingest/backfill_mbin_test.go b/internal/ingest/backfill_mbin_test.go new file mode 100644 index 0000000..48ed418 --- /dev/null +++ b/internal/ingest/backfill_mbin_test.go @@ -0,0 +1,116 @@ +package ingest + +import ( + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "tidepool/internal/ap" + "tidepool/internal/materialize" +) + +// Mbin link post fixture ids, rehomed into the harness's subscribed +// lemmy.world/c/technology community. +const ( + mbinPageID = "https://fedia.io/m/technology@lemmy.world/t/4194502" + mbinAuthorID = "https://fedia.io/u/troed" + mbinSourceURL = "https://video.troed.se/w/5FqDa2jVqCcQWxetJEivnw" +) + +// serveMbinLinkPage serves the Mbin link Page as raw JSON — never +// round-tripped through ap.Object, which would normalize away the wire shape +// under test — rehomed from the fixture's startrek.website community into the +// harness's subscribed technology community, plus its Mbin author. +func serveMbinLinkPage(t *testing.T, h *harness) { + t.Helper() + page := loadFixture(t, "page_mbin_fedia_io.json") + require.Equal(t, mbinSourceURL, page["source"], + "precondition: the fixture carries Mbin's bare-URL-string source") + page["id"] = mbinPageID + page["url"] = mbinPageID + page["audience"] = groupID + page["to"] = []any{groupID, ap.PublicAudience} + page["cc"] = []any{mbinAuthorID + "/followers"} + h.serveObject(urlPath(t, mbinPageID), page) + h.serveObject("/u/troed", person(mbinAuthorID, "troed", nil)) +} + +// serveMbinOutbox replaces the technology outbox with a single Lemmy-style +// Announce{Create} whose Page is referenced BY IRI, so the backfill has to +// fetch the Mbin Page from its origin. +func serveMbinOutbox(h *harness) { + h.serveObject("/c/technology/outbox", map[string]any{ + "type": "OrderedCollection", + "id": "https://lemmy.world/c/technology/outbox", + "totalItems": 1, + "orderedItems": []any{ + map[string]any{ + "type": "Announce", + "id": "https://lemmy.world/activities/announce/create/mbin-4194502", + "actor": groupID, + "to": []any{ap.PublicAudience}, + "cc": []any{groupID + "/followers"}, + "object": map[string]any{ + "type": "Create", + "id": "https://fedia.io/f/object/7c1b2f4e-3d5a-4b8e-9f10-2a6c8d4e5b71", + "actor": mbinAuthorID, + "to": []any{groupID, ap.PublicAudience}, + "cc": []any{mbinAuthorID + "/followers"}, + "audience": groupID, + "object": mbinPageID, + }, + }, + }, + }) +} + +// TestBackfillMaterializesMbinLinkPageWithStringSource: Mbin (fedia.io) +// serves a link post as a Page whose `source` is the bare external URL string +// rather than Lemmy's {content, mediaType} markdown object. A backfill that +// meets such a Page in a subscribed community's outbox must materialize it — +// not fail the item on the parse — and the post body must come from the +// Page's HTML `content`: the URL string is never markdown source. +// +// The Page is hand-built from Mbin's EntryPageFactory output shape (no live +// fetch): internal/ap/testdata/page_mbin_fedia_io.json. +func TestBackfillMaterializesMbinLinkPageWithStringSource(t *testing.T) { + h := newHarness(t) + h.subscribeTechnology() + serveMbinLinkPage(t, h) + serveMbinOutbox(h) + // The run's error only counts failed items; the per-item cause is logged, + // so the log is captured to name it when this fails. + logs := &syncBuffer{} + b, err := NewBackfill(BackfillOptions{ + Fetcher: h.client, + Materializer: h.mat, + Communities: h.communities, + Tombstones: h.tombstones, + Echo: h.classifier, + MaxPosts: 10, + Logger: slog.New(slog.NewTextHandler(logs, &slog.HandlerOptions{Level: slog.LevelDebug})), + }) + require.NoError(t, err) + ctx := context.Background() + + community, err := h.communities.GetByAPGroupID(ctx, groupID) + require.NoError(t, err) + err = b.Run(ctx, community, true) + require.NoError(t, err, + "an Mbin Page with a string source must not fail the backfill item; backfill log:\n%s", logs.String()) + + mapping, err := h.objects.GetByAPID(ctx, mbinPageID) + require.NoError(t, err, "the Mbin link post must be materialized") + assert.Equal(t, materialize.CollectionPostV2, mapping.Collection) + + record, _, err := h.manager.GetRecord(ctx, mapping.DID, mapping.Collection, mapping.RKey) + require.NoError(t, err) + content, _ := record["content"].(string) + assert.Equal(t, "Mbin link post body text", content, + "the body is the plaintext of the Page's HTML content") + assert.NotContains(t, content, mbinSourceURL, + "the string source is a URL, never the post's markdown body") +}