From 9dbb48e60c4464ac7095b0e70f88b0b4a52de07d Mon Sep 17 00:00:00 2001 From: Johanna Larsson Date: Sun, 21 Jun 2026 09:05:46 +0000 Subject: [PATCH] Adds feed discovery, parsing, and some plumbing for sites Feed discovery is limited to link rel=alternate, which is probably plenty? It could be extended with some "guessing", checking if `/rss.xml` returns a valid feed, but that can be a future thing. Probably going to want to make it possible to enable/disable feed polling, which might live on site, but we'll figure that out when we get to it --- lib/annot_at/feeds.ex | 59 +++++++++++++++++++++++++++++++++++++---------------------- lib/annot_at/publishing.ex | 44 ++++++++++++++++++++++++++++++++++++++++++++ test/annot_at/feeds_test.exs | 59 ++++++++++++++++++++++++++++++++++++++++------------------- test/annot_at/publishing_test.exs | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ lib/annot_at/feeds/source.ex | 16 ++++++++++++++++ lib/annot_at/publishing/site.ex | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ priv/repo/migrations/20260620150821_create_sites.exs | 19 +++++++++++++++++++ 7 file(s) changed, 283 insertion(s)(+), 41 deletion(s)(-) diff --git a/lib/annot_at/feeds.ex b/lib/annot_at/feeds.ex --- a/lib/annot_at/feeds.ex +++ b/lib/annot_at/feeds.ex @@ -6,8 +6,17 @@ alias AnnotAt.Feeds.Feed alias AnnotAt.Feeds.RSS + alias AnnotAt.Feeds.Source - @feed_types ~w(application/rss+xml application/atom+xml appliation/feed+json) + @feed_formats %{ + "application/rss+xml" => :rss, + "application/atom+xml" => :atom, + "application/feed+json" => :json + } + + @feed_selector Enum.map_join(Map.keys(@feed_formats), ", ", fn type -> + ~s(link[rel="alternate"][type="#{type}"]) + end) @doc """ Parses a feed body into a `Feed`, detecting the format from body and @@ -25,33 +34,39 @@ end @doc """ - Finds a feed URL on a page. + Finds every feed URL on a page. - Looks for link alternate pointing to a feed and returns the first match. + Looks for link alternate pointing to feeds and returns all matches. """ - @spec discover(binary(), String.t()) :: {:ok, String.t()} | :error + @spec discover(binary(), String.t()) :: [Source.t()] def discover(html, base_url) when is_binary(html) and is_binary(base_url) do - selector = - Enum.map_join(@feed_types, ", ", fn type -> - ~s(link[rel="alternate"][type="#{type}"]) - end) + html + |> LazyHTML.from_document() + |> LazyHTML.query(@feed_selector) + |> LazyHTML.attributes() + |> Enum.map(&source_from_attributes(&1, base_url)) + |> Enum.reject(&is_nil/1) + |> Enum.uniq_by(& &1.url) + end - href = - html - |> LazyHTML.from_document() - |> LazyHTML.query(selector) - |> LazyHTML.attribute("href") - |> List.first() + defp source_from_attributes(attributes, base_url) do + attributes = Map.new(attributes) - if href do - url = - base_url - |> URI.merge(href) - |> URI.to_string() + case attributes do + %{"href" => href, "type" => type} -> + url = + base_url + |> URI.merge(href) + |> URI.to_string() - {:ok, url} - else - :error + %Source{ + url: url, + title: attributes["title"], + format: Map.fetch!(@feed_formats, type) + } + + _ -> + nil end end diff --git a/lib/annot_at/publishing.ex b/lib/annot_at/publishing.ex new file mode 100644 --- /dev/null +++ b/lib/annot_at/publishing.ex @@ -0,0 +1,44 @@ +defmodule AnnotAt.Publishing do + @moduledoc """ + Context for a user's sites. Each site mirrors a `site.standard.publication` + record in the user's atproto rep. Each user can have many sites, one for + each actual website they control. Only created after verification. + """ + + import Ecto.Query, only: [from: 2] + + alias AnnotAt.Accounts.Scope + alias AnnotAt.Accounts.User + alias AnnotAt.Publishing.Site + alias AnnotAt.Repo + + @spec list_sites(Scope.t()) :: [Site.t()] + def list_sites(%Scope{user: %User{id: user_id}}) do + Repo.all(from s in Site, where: s.user_id == ^user_id, order_by: [asc: s.inserted_at]) + end + + def get_site!(%Scope{user: %User{id: user_id}}, id) do + Repo.get_by!(Site, id: id, user_id: user_id) + end + + def create_site(%Scope{user: %User{id: user_id}}, rkey, attrs) do + %Site{ + user_id: user_id, + rkey: rkey, + verified_at: DateTime.utc_now(:second) + } + |> Site.changeset(attrs) + |> Repo.insert() + end + + def update_site(%Scope{user: %User{id: user_id}}, %Site{} = site, attrs) do + verify_user_ownership!(site, user_id) + + site + |> Site.changeset(attrs) + |> Repo.update() + end + + defp verify_user_ownership!(%Site{user_id: user_id}, user_id), do: :ok + defp verify_user_ownership!(%Site{}, _user_id), do: raise(Ecto.NoResultsError, queryable: Site) +end diff --git a/test/annot_at/feeds_test.exs b/test/annot_at/feeds_test.exs --- a/test/annot_at/feeds_test.exs +++ b/test/annot_at/feeds_test.exs @@ -19,31 +19,52 @@ end describe "Feeds.discover/2" do - test "finds and resolves a relative feed url" do - html = """ - - - - - + test "returns all declared feeds, resolved and labeled" do + html = ~s""" + + + + + """ - assert {:ok, "https://blog.example.com/feed.xml"} = + assert [main, atom, json] = Feeds.discover(html, "https://blog.example.com") + + assert %Feeds.Source{url: "https://blog.example.com/feed.xml", title: "Main", format: :rss} = + main + + assert %Feeds.Source{url: "https://blog.example.com/atom", title: nil, format: :atom} = atom + assert %Feeds.Source{format: :json, title: "JSON"} = json + end + + test "ignores non-feed alternates and dedupes by url" do + html = ~s""" + + + + + + """ + + assert [%Feeds.Source{url: "https://example.com/feed.xml"}] = + Feeds.discover(html, "https://example.com") + end + + test "returns [] when no feed is declared" do + assert [] == Feeds.discover( - html, - "https://blog.example.com" + "", + "https://example.com" ) end - test "returns :error when there's no feed link" do - html = """ - - - - - """ - - assert :error = Feeds.discover(html, "https://blog.example.com") + test "skips feed links without an href" do + html = ~s() + assert [] == Feeds.discover(html, "https://example.com") end end end diff --git a/test/annot_at/publishing_test.exs b/test/annot_at/publishing_test.exs new file mode 100644 --- /dev/null +++ b/test/annot_at/publishing_test.exs @@ -0,0 +1,79 @@ +defmodule AnnotAt.PublishingTest do + use AnnotAt.DataCase, async: true + + alias AnnotAt.Accounts + alias AnnotAt.Accounts.Scope + alias AnnotAt.Publishing + alias AnnotAt.Publishing.Site + + setup do + {:ok, user} = + Accounts.upsert_user(%{ + did: "did:plc:ewvi7nxzyoun6zhxrhs64oiz", + handle: "alice.test", + pds_host: "https://pds.example.com" + }) + + %{scope: Scope.for_user(user)} + end + + test "create_site/3 persists a verified site owned by the scope", %{scope: scope} do + assert {:ok, %Site{} = site} = Publishing.create_site(scope, "3mope7jyypk22", site_attrs()) + assert site.user_id == scope.user.id + assert "3mope7jyypk22" == site.rkey + assert %DateTime{} = site.verified_at + + assert [listed] = Publishing.list_sites(scope) + assert listed.id == site.id + end + + test "a user can hold many sites for different websites", %{scope: scope} do + {:ok, _} = Publishing.create_site(scope, "aaa", site_attrs(%{url: "https://one.com"})) + {:ok, _} = Publishing.create_site(scope, "bbb", site_attrs(%{url: "https://two.com"})) + + assert 2 == length(Publishing.list_sites(scope)) + end + + test "create_site/3 requires name, url and feed_url", %{scope: scope} do + assert {:error, changeset} = Publishing.create_site(scope, "rkey", %{}) + assert %{name: _, url: _, feed_url: _} = errors_on(changeset) + end + + test "rkey is unique per user", %{scope: scope} do + {:ok, _} = Publishing.create_site(scope, "dup", site_attrs()) + assert {:error, changeset} = Publishing.create_site(scope, "dup", site_attrs()) + assert %{rkey: _} = errors_on(changeset) + end + + test "get_site!/2 raises for a site the scope doesn't own", %{scope: scope} do + {:ok, other} = Accounts.upsert_user(%{did: "did:plc:otheruser000000000000000"}) + other_scope = Scope.for_user(other) + {:ok, site} = Publishing.create_site(scope, "scoped", site_attrs()) + + assert Publishing.get_site!(scope, site.id).id == site.id + + assert_raise Ecto.NoResultsError, fn -> + Publishing.get_site!( + other_scope, + site.id + ) + end + end + + test "update_site/3 refuses a site the scope doesn't own", %{scope: scope} do + {:ok, other} = Accounts.upsert_user(%{did: "did:plc:otheruser000000000000000"}) + other_scope = Scope.for_user(other) + {:ok, site} = Publishing.create_site(scope, "owned", site_attrs()) + + assert_raise Ecto.NoResultsError, fn -> + Publishing.update_site(other_scope, site, %{name: "x"}) + end + end + + defp site_attrs(overrides \\ %{}) do + Map.merge( + %{name: "My Blog", url: "https://example.com", feed_url: "https://example.com/feed.xml"}, + overrides + ) + end +end diff --git a/lib/annot_at/feeds/source.ex b/lib/annot_at/feeds/source.ex new file mode 100644 --- /dev/null +++ b/lib/annot_at/feeds/source.ex @@ -0,0 +1,16 @@ +defmodule AnnotAt.Feeds.Source do + @moduledoc """ + A nice wrapper for links to feeds containing metadata about them. + """ + + @type format :: :rss | :atom | :json + + @type t :: %__MODULE__{ + url: String.t(), + title: String.t() | nil, + format: format() + } + + @enforce_keys [:url, :format] + defstruct [:url, :title, :format] +end diff --git a/lib/annot_at/publishing/site.ex b/lib/annot_at/publishing/site.ex new file mode 100644 --- /dev/null +++ b/lib/annot_at/publishing/site.ex @@ -0,0 +1,48 @@ +defmodule AnnotAt.Publishing.Site do + use Ecto.Schema + + import Ecto.Changeset + + @type t :: %__MODULE__{ + user_id: integer(), + name: String.t(), + url: String.t(), + description: String.t() | nil, + feed_url: String.t(), + rkey: String.t(), + verified_at: DateTime.t() + } + + schema "sites" do + # Display name of the publication + field :name, :string + # The website the publication represents + field :url, :string + # Tagline + field :description, :string + # The url of the feed + field :feed_url, :string + # rkey of the site.standard.publication record + # deterministically generated from did and url + field :rkey, :string + # when the site was verified + field :verified_at, :utc_datetime + + belongs_to :user, AnnotAt.Accounts.User + + timestamps(type: :utc_datetime) + end + + def changeset(site, attrs) do + site + |> cast(attrs, [:name, :url, :description, :feed_url]) + |> validate_required([:name, :url, :feed_url, :rkey]) + |> validate_length(:name, max: 255) + |> validate_length(:description, max: 1000) + |> validate_length(:url, max: 2048) + |> validate_length(:feed_url, max: 2048) + |> validate_length(:rkey, max: 512) + |> foreign_key_constraint(:user_id) + |> unique_constraint(:rkey, name: :sites_user_id_rkey_index) + end +end diff --git a/priv/repo/migrations/20260620150821_create_sites.exs b/priv/repo/migrations/20260620150821_create_sites.exs new file mode 100644 --- /dev/null +++ b/priv/repo/migrations/20260620150821_create_sites.exs @@ -0,0 +1,19 @@ +defmodule AnnotAt.Repo.Migrations.CreateSites do + use Ecto.Migration + + def change do + create table(:sites) do + add :user_id, references(:users, on_delete: :delete_all), null: false + add :name, :text, null: false + add :url, :text, null: false + add :description, :text + add :feed_url, :text, null: false + add :rkey, :text, null: false + add :verified_at, :utc_datetime, null: false + + timestamps(type: :utc_datetime) + end + + create unique_index(:sites, [:user_id, :rkey]) + end +end -- tangled.sh