diff --git a/.gitignore b/.gitignore index da335b3..e7993d2 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ npm-debug.log /assets/node_modules/ .mix_tasks +.DS_Store diff --git a/config/config.exs b/config/config.exs index fb4e9fe..807b561 100644 --- a/config/config.exs +++ b/config/config.exs @@ -55,7 +55,7 @@ config :tailwind, # Configure Elixir's Logger config :logger, :default_formatter, format: "$time $metadata[$level] $message\n", - metadata: [:request_id] + metadata: :all # Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason diff --git a/lib/annot_at/accounts/user.ex b/lib/annot_at/accounts/user.ex index 3696af1..1e96852 100644 --- a/lib/annot_at/accounts/user.ex +++ b/lib/annot_at/accounts/user.ex @@ -3,6 +3,15 @@ defmodule AnnotAt.Accounts.User do import Ecto.Changeset + @type t :: %__MODULE__{ + did: String.t(), + handle: String.t(), + display_name: String.t(), + avatar_url: String.t(), + pds_host: String.t(), + handle_verified_at: DateTime.t() + } + schema "users" do # Stable atproto identity from OAuth sub field :did, :string diff --git a/lib/annot_at/atproto/profile.ex b/lib/annot_at/atproto/profile.ex index f53abe8..4ebdf1a 100644 --- a/lib/annot_at/atproto/profile.ex +++ b/lib/annot_at/atproto/profile.ex @@ -13,7 +13,6 @@ defmodule AnnotAt.Atproto.Profile do @spec fetch(String.t()) :: {:ok, %{display_name: String.t() | nil, avatar: String.t() | nil}} | {:error, {:http_status, pos_integer()} | {:transport, term()} | :invalid_json} - def fetch(actor) do url = "#{@appview}/xrpc/app.bsky.actor.getProfile?#{URI.encode_query(actor: actor)}" diff --git a/lib/annot_at/feeds.ex b/lib/annot_at/feeds.ex new file mode 100644 index 0000000..620788a --- /dev/null +++ b/lib/annot_at/feeds.ex @@ -0,0 +1,72 @@ +defmodule AnnotAt.Feeds do + @moduledoc """ + Feed handling entry point. Discovers feed URLs from a page HTML, + detects format and parses. + """ + + alias AnnotAt.Feeds.Feed + alias AnnotAt.Feeds.RSS + + @feed_types ~w(application/rss+xml application/atom+xml appliation/feed+json) + + @doc """ + Parses a feed body into a `Feed`, detecting the format from body and + content type. + """ + @spec parse(binary(), String.t() | nil) :: + {:ok, Feed.t()} | {:error, :invalid_feed | :unsupported_feed | :unrecognized_feed} + def parse(body, content_type \\ nil) when is_binary(body) do + case detect(body, content_type) do + :rss -> RSS.parse(body) + :atom -> {:error, :unsupported_feed} + :json -> {:error, :unsupported_feed} + :unknown -> {:error, :unrecognized_feed} + end + end + + @doc """ + Finds a feed URL on a page. + + Looks for link alternate pointing to a feed and returns the first match. + """ + @spec discover(binary(), String.t()) :: {:ok, String.t()} | :error + def discover(html, base_url) when is_binary(html) and is_binary(base_url) do + selector = + Enum.map_join(@feed_types, ", ", fn type -> + ~s(link[rel="alternate"][type="#{type}"]) + end) + + href = + html + |> LazyHTML.from_document() + |> LazyHTML.query(selector) + |> LazyHTML.attribute("href") + |> List.first() + + if href do + url = + base_url + |> URI.merge(href) + |> URI.to_string() + + {:ok, url} + else + :error + end + end + + defp detect(body, content_type) do + head = + body + |> String.slice(0..1023) + |> String.trim_leading() + + cond do + String.starts_with?(head, "{") -> :json + is_binary(content_type) and String.contains?(content_type, "json") -> :json + String.contains?(head, " :rss + String.contains?(head, " :atom + true -> :unknown + end + end +end diff --git a/lib/annot_at/feeds/entry.ex b/lib/annot_at/feeds/entry.ex new file mode 100644 index 0000000..d6c09e0 --- /dev/null +++ b/lib/annot_at/feeds/entry.ex @@ -0,0 +1,19 @@ +defmodule AnnotAt.Feeds.Entry do + @moduledoc """ + A normalized feed entry, format-agnostic. + + Used as a normalized feed entry format, where entries come from + different forms like RSS and atom. + """ + + defstruct [:id, :url, :title, :published_at, :summary, :content] + + @type t :: %__MODULE__{ + id: String.t() | nil, + url: String.t() | nil, + title: String.t() | nil, + published_at: DateTime.t() | nil, + summary: String.t() | nil, + content: String.t() | nil + } +end diff --git a/lib/annot_at/feeds/feed.ex b/lib/annot_at/feeds/feed.ex new file mode 100644 index 0000000..80fb720 --- /dev/null +++ b/lib/annot_at/feeds/feed.ex @@ -0,0 +1,18 @@ +defmodule AnnotAt.Feeds.Feed do + @moduledoc """ + Normalized feed metadata plus entries, format-agnostic. + + Note that the `url` field is the channel's canonical URL. + """ + + alias AnnotAt.Feeds.Entry + + defstruct [:title, :description, :url, entries: []] + + @type t :: %__MODULE__{ + title: String.t(), + description: String.t() | nil, + url: String.t() | nil, + entries: [Entry.t()] + } +end diff --git a/lib/annot_at/feeds/rss.ex b/lib/annot_at/feeds/rss.ex new file mode 100644 index 0000000..e79080f --- /dev/null +++ b/lib/annot_at/feeds/rss.ex @@ -0,0 +1,146 @@ +defmodule AnnotAt.Feeds.RSS do + @moduledoc """ + Saxy SAX parser for RSS 2.0 feeds into `AnnotAt.Feeds.Feed`. + + Used Gluttony as a reference implementation. + """ + + @behaviour Saxy.Handler + + alias AnnotAt.Feeds.Entry + alias AnnotAt.Feeds.Feed + + require Logger + + @doc """ + Returns `AnnotAt.Feeds.Feed` with entries. The list of entries can be empty. + + If the feed is invalid or not usable, it returns `{:error, :invalid_feed}`. + """ + @spec parse(binary()) :: {:ok, Feed.t()} | {:error, :invalid_feed} + def parse(body) when is_binary(body) do + case Saxy.parse_string(body, __MODULE__, initial_state()) do + {:ok, state} -> + feed = %{state.feed | entries: Enum.reverse(state.entries)} + + if is_binary(feed.title) do + {:ok, feed} + else + {:error, :invalid_feed} + end + + {:error, %Saxy.ParseError{} = saxy_error} -> + Logger.warning("Feeds.RSS saxy error", + error: inspect(saxy_error) + ) + + {:error, :invalid_feed} + end + end + + defp initial_state do + %{ + feed: %Feed{}, + entries: [], + stack: [], + current_text: [] + } + end + + def handle_event(:start_document, _data, state), do: {:ok, state} + + def handle_event(:start_element, {"item", _attrs}, state) do + {:ok, + %{ + state + | entries: [%Entry{} | state.entries], + stack: ["item" | state.stack], + current_text: [] + }} + end + + def handle_event(:start_element, {name, _attrs}, state) do + {:ok, %{state | stack: [name | state.stack], current_text: []}} + end + + def handle_event(:characters, chars, state) do + {:ok, %{state | current_text: [chars | state.current_text]}} + end + + def handle_event(:end_element, "item", state) do + ["item" | rest_stack] = state.stack + [entry | rest] = state.entries + entry = finalize_entry(entry) + {:ok, %{state | entries: [entry | rest], stack: rest_stack, current_text: []}} + end + + def handle_event(:end_element, name, state) do + text = text(state.current_text) + [^name | parent_stack] = state.stack + parent = List.first(parent_stack) + + state = %{state | stack: parent_stack, current_text: []} + + state = + cond do + parent == "item" -> + [current | entries] = state.entries + %{state | entries: [apply_entry_field(current, name, text) | entries]} + + parent == "channel" -> + %{state | feed: apply_feed_field(state.feed, name, text)} + + true -> + state + end + + {:ok, state} + end + + def handle_event(:end_document, _data, state), do: {:ok, state} + + defp apply_entry_field(entry, "title", text), do: %{entry | title: text} + defp apply_entry_field(entry, "link", text), do: %{entry | url: text} + defp apply_entry_field(entry, "guid", text), do: %{entry | id: text} + defp apply_entry_field(entry, "description", text), do: %{entry | summary: text} + defp apply_entry_field(entry, "content:encoded", text), do: %{entry | content: text} + defp apply_entry_field(entry, "pubDate", text), do: %{entry | published_at: parse_date(text)} + defp apply_entry_field(entry, _name, _text), do: entry + + defp apply_feed_field(feed, "title", text), do: %{feed | title: text} + defp apply_feed_field(feed, "description", text), do: %{feed | description: text} + defp apply_feed_field(feed, "link", text), do: %{feed | url: text} + defp apply_feed_field(feed, _name, _text), do: feed + + defp finalize_entry(%Entry{id: nil, url: url} = entry) when is_binary(url) do + %{entry | id: url} + end + + defp finalize_entry(entry), do: entry + + defp text(parts) do + result = + parts + |> Enum.reverse() + |> IO.iodata_to_binary() + |> String.trim() + + case result do + "" -> nil + trimmed -> trimmed + end + end + + defp parse_date(nil), do: nil + + defp parse_date(text) do + case DateTimeParser.parse_datetime(text) do + {:ok, datetime} -> + datetime + + {:error, reason} -> + Logger.debug("Feeds.RSS unparseable pubDate #{inspect(text)}: #{inspect(reason)}") + nil + end + end +end diff --git a/lib/annot_at_web/components/layouts/root.html.heex b/lib/annot_at_web/components/layouts/root.html.heex index 64e2aad..0b7592d 100644 --- a/lib/annot_at_web/components/layouts/root.html.heex +++ b/lib/annot_at_web/components/layouts/root.html.heex @@ -33,7 +33,6 @@ /> - Path.expand(__DIR__) + |> File.read!() + + test "parses channel-level feed metadata" do + assert {:ok, %Feed{} = feed} = RSS.parse(@fixture) + assert "Sample Blog" == feed.title + assert "https://example.com" == feed.url + assert "Thoughts about things." == feed.description + end + + test "parses entries with all fields" do + assert {:ok, %{entries: [first, _second]}} = RSS.parse(@fixture) + + assert %Entry{} = first + assert "First Post" == first.title + assert "https://example.com/posts/first" == first.url + assert "abc" == first.id + assert "A short summary of the first post." == first.summary + assert "

The full content of the first post.

" == first.content + assert %DateTime{} = first.published_at + assert ~U[2024-10-02 13:00:00Z] == first.published_at + end + + test "falls back to URL as ID when guid is missing" do + assert {:ok, %{entries: [_first, second]}} = RSS.parse(@fixture) + assert second.id == second.url + end + + test "leaves published_at nil when pubDate is missing" do + assert {:ok, %{entries: [_first, second]}} = RSS.parse(@fixture) + refute second.published_at + end + + test "leaves content nil when missing" do + assert {:ok, %{entries: [_first, second]}} = RSS.parse(@fixture) + refute second.content + end + + test "returns an error on malformed" do + assert {:error, :invalid_feed} = RSS.parse("oops") + end + + test "accepts a valid but empty blog" do + body = """ + <?xml version="1.0" encoding="UTF-8"?> + <rss version="2.0"> + <channel> + <title>Brand New Blog + https://example.com + Nothing here yet. + + + """ + + assert {:ok, %Feed{title: "Brand New Blog", entries: []}} = RSS.parse(body) + end + + test "rejects a feed with no channel title as invalid" do + body = """ + + + + https://example.com + No title here. + + + """ + + assert {:error, :invalid_feed} = RSS.parse(body) + end +end diff --git a/test/annot_at/feeds_test.exs b/test/annot_at/feeds_test.exs new file mode 100644 index 0000000..7287f07 --- /dev/null +++ b/test/annot_at/feeds_test.exs @@ -0,0 +1,49 @@ +defmodule AnnotAt.FeedsTest do + use ExUnit.Case, async: true + + alias AnnotAt.Feeds + alias AnnotAt.Feeds.Feed + + @fixture "../support/fixtures/feeds/rss_sample.xml" + |> Path.expand(__DIR__) + |> File.read!() + + describe "Feeds.parse/2" do + test "detects and dispatches RSS" do + assert {:ok, %Feed{title: "Sample Blog"}} = Feeds.parse(@fixture, "application/rss+xml") + end + + test "rejects an unrecognized body" do + assert {:error, :unrecognized_feed} = Feeds.parse("not a feed at all", nil) + end + end + + describe "Feeds.discover/2" do + test "finds and resolves a relative feed url" do + html = """ + + + + + + """ + + assert {:ok, "https://blog.example.com/feed.xml"} = + Feeds.discover( + html, + "https://blog.example.com" + ) + end + + test "returns :error when there's no feed link" do + html = """ + + + + + """ + + assert :error = Feeds.discover(html, "https://blog.example.com") + end + end +end diff --git a/test/support/fixtures/feeds/rss_sample.xml b/test/support/fixtures/feeds/rss_sample.xml new file mode 100644 index 0000000..b6c8613 --- /dev/null +++ b/test/support/fixtures/feeds/rss_sample.xml @@ -0,0 +1,27 @@ + + + + Sample Blog + https://example.com + Thoughts about things. + + https://example.com/icon.png + Sample Blog Logo + https://example.com + + + First Post + https://example.com/posts/first + abc + A short summary of the first post. + The full content of the first post.

]]>
+ Wed, 02 Oct 2024 13:00:00 GMT +
+ + Second Post + https://example.com/posts/second + Another summary. + +
+