diff --git a/.gitignore b/.gitignore
index da335b3..e7993d2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,3 +36,4 @@ npm-debug.log
/assets/node_modules/
.mix_tasks
+.DS_Store
diff --git a/config/config.exs b/config/config.exs
index fb4e9fe..807b561 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -55,7 +55,7 @@ config :tailwind,
# Configure Elixir's Logger
config :logger, :default_formatter,
format: "$time $metadata[$level] $message\n",
- metadata: [:request_id]
+ metadata: :all
# Use Jason for JSON parsing in Phoenix
config :phoenix, :json_library, Jason
diff --git a/lib/annot_at/accounts/user.ex b/lib/annot_at/accounts/user.ex
index 3696af1..1e96852 100644
--- a/lib/annot_at/accounts/user.ex
+++ b/lib/annot_at/accounts/user.ex
@@ -3,6 +3,15 @@ defmodule AnnotAt.Accounts.User do
import Ecto.Changeset
+ @type t :: %__MODULE__{
+ did: String.t(),
+ handle: String.t(),
+ display_name: String.t(),
+ avatar_url: String.t(),
+ pds_host: String.t(),
+ handle_verified_at: DateTime.t()
+ }
+
schema "users" do
# Stable atproto identity from OAuth sub
field :did, :string
diff --git a/lib/annot_at/atproto/profile.ex b/lib/annot_at/atproto/profile.ex
index f53abe8..4ebdf1a 100644
--- a/lib/annot_at/atproto/profile.ex
+++ b/lib/annot_at/atproto/profile.ex
@@ -13,7 +13,6 @@ defmodule AnnotAt.Atproto.Profile do
@spec fetch(String.t()) ::
{:ok, %{display_name: String.t() | nil, avatar: String.t() | nil}}
| {:error, {:http_status, pos_integer()} | {:transport, term()} | :invalid_json}
-
def fetch(actor) do
url = "#{@appview}/xrpc/app.bsky.actor.getProfile?#{URI.encode_query(actor: actor)}"
diff --git a/lib/annot_at/feeds.ex b/lib/annot_at/feeds.ex
new file mode 100644
index 0000000..620788a
--- /dev/null
+++ b/lib/annot_at/feeds.ex
@@ -0,0 +1,72 @@
+defmodule AnnotAt.Feeds do
+ @moduledoc """
+ Feed handling entry point. Discovers feed URLs from a page HTML,
+ detects format and parses.
+ """
+
+ alias AnnotAt.Feeds.Feed
+ alias AnnotAt.Feeds.RSS
+
+ @feed_types ~w(application/rss+xml application/atom+xml appliation/feed+json)
+
+ @doc """
+ Parses a feed body into a `Feed`, detecting the format from body and
+ content type.
+ """
+ @spec parse(binary(), String.t() | nil) ::
+ {:ok, Feed.t()} | {:error, :invalid_feed | :unsupported_feed | :unrecognized_feed}
+ def parse(body, content_type \\ nil) when is_binary(body) do
+ case detect(body, content_type) do
+ :rss -> RSS.parse(body)
+ :atom -> {:error, :unsupported_feed}
+ :json -> {:error, :unsupported_feed}
+ :unknown -> {:error, :unrecognized_feed}
+ end
+ end
+
+ @doc """
+ Finds a feed URL on a page.
+
+ Looks for link alternate pointing to a feed and returns the first match.
+ """
+ @spec discover(binary(), String.t()) :: {:ok, String.t()} | :error
+ def discover(html, base_url) when is_binary(html) and is_binary(base_url) do
+ selector =
+ Enum.map_join(@feed_types, ", ", fn type ->
+ ~s(link[rel="alternate"][type="#{type}"])
+ end)
+
+ href =
+ html
+ |> LazyHTML.from_document()
+ |> LazyHTML.query(selector)
+ |> LazyHTML.attribute("href")
+ |> List.first()
+
+ if href do
+ url =
+ base_url
+ |> URI.merge(href)
+ |> URI.to_string()
+
+ {:ok, url}
+ else
+ :error
+ end
+ end
+
+ defp detect(body, content_type) do
+ head =
+ body
+ |> String.slice(0..1023)
+ |> String.trim_leading()
+
+ cond do
+ String.starts_with?(head, "{") -> :json
+ is_binary(content_type) and String.contains?(content_type, "json") -> :json
+ String.contains?(head, " :rss
+ String.contains?(head, " :atom
+ true -> :unknown
+ end
+ end
+end
diff --git a/lib/annot_at/feeds/entry.ex b/lib/annot_at/feeds/entry.ex
new file mode 100644
index 0000000..d6c09e0
--- /dev/null
+++ b/lib/annot_at/feeds/entry.ex
@@ -0,0 +1,19 @@
+defmodule AnnotAt.Feeds.Entry do
+ @moduledoc """
+ A normalized feed entry, format-agnostic.
+
+ Used as a normalized feed entry format, where entries come from
+ different forms like RSS and atom.
+ """
+
+ defstruct [:id, :url, :title, :published_at, :summary, :content]
+
+ @type t :: %__MODULE__{
+ id: String.t() | nil,
+ url: String.t() | nil,
+ title: String.t() | nil,
+ published_at: DateTime.t() | nil,
+ summary: String.t() | nil,
+ content: String.t() | nil
+ }
+end
diff --git a/lib/annot_at/feeds/feed.ex b/lib/annot_at/feeds/feed.ex
new file mode 100644
index 0000000..80fb720
--- /dev/null
+++ b/lib/annot_at/feeds/feed.ex
@@ -0,0 +1,18 @@
+defmodule AnnotAt.Feeds.Feed do
+ @moduledoc """
+ Normalized feed metadata plus entries, format-agnostic.
+
+ Note that the `url` field is the channel's canonical URL.
+ """
+
+ alias AnnotAt.Feeds.Entry
+
+ defstruct [:title, :description, :url, entries: []]
+
+ @type t :: %__MODULE__{
+ title: String.t(),
+ description: String.t() | nil,
+ url: String.t() | nil,
+ entries: [Entry.t()]
+ }
+end
diff --git a/lib/annot_at/feeds/rss.ex b/lib/annot_at/feeds/rss.ex
new file mode 100644
index 0000000..e79080f
--- /dev/null
+++ b/lib/annot_at/feeds/rss.ex
@@ -0,0 +1,146 @@
+defmodule AnnotAt.Feeds.RSS do
+ @moduledoc """
+ Saxy SAX parser for RSS 2.0 feeds into `AnnotAt.Feeds.Feed`.
+
+ Used Gluttony as a reference implementation.
+ """
+
+ @behaviour Saxy.Handler
+
+ alias AnnotAt.Feeds.Entry
+ alias AnnotAt.Feeds.Feed
+
+ require Logger
+
+ @doc """
+ Returns `AnnotAt.Feeds.Feed` with entries. The list of entries can be empty.
+
+ If the feed is invalid or not usable, it returns `{:error, :invalid_feed}`.
+ """
+ @spec parse(binary()) :: {:ok, Feed.t()} | {:error, :invalid_feed}
+ def parse(body) when is_binary(body) do
+ case Saxy.parse_string(body, __MODULE__, initial_state()) do
+ {:ok, state} ->
+ feed = %{state.feed | entries: Enum.reverse(state.entries)}
+
+ if is_binary(feed.title) do
+ {:ok, feed}
+ else
+ {:error, :invalid_feed}
+ end
+
+ {:error, %Saxy.ParseError{} = saxy_error} ->
+ Logger.warning("Feeds.RSS saxy error",
+ error: inspect(saxy_error)
+ )
+
+ {:error, :invalid_feed}
+ end
+ end
+
+ defp initial_state do
+ %{
+ feed: %Feed{},
+ entries: [],
+ stack: [],
+ current_text: []
+ }
+ end
+
+ def handle_event(:start_document, _data, state), do: {:ok, state}
+
+ def handle_event(:start_element, {"item", _attrs}, state) do
+ {:ok,
+ %{
+ state
+ | entries: [%Entry{} | state.entries],
+ stack: ["item" | state.stack],
+ current_text: []
+ }}
+ end
+
+ def handle_event(:start_element, {name, _attrs}, state) do
+ {:ok, %{state | stack: [name | state.stack], current_text: []}}
+ end
+
+ def handle_event(:characters, chars, state) do
+ {:ok, %{state | current_text: [chars | state.current_text]}}
+ end
+
+ def handle_event(:end_element, "item", state) do
+ ["item" | rest_stack] = state.stack
+ [entry | rest] = state.entries
+ entry = finalize_entry(entry)
+ {:ok, %{state | entries: [entry | rest], stack: rest_stack, current_text: []}}
+ end
+
+ def handle_event(:end_element, name, state) do
+ text = text(state.current_text)
+ [^name | parent_stack] = state.stack
+ parent = List.first(parent_stack)
+
+ state = %{state | stack: parent_stack, current_text: []}
+
+ state =
+ cond do
+ parent == "item" ->
+ [current | entries] = state.entries
+ %{state | entries: [apply_entry_field(current, name, text) | entries]}
+
+ parent == "channel" ->
+ %{state | feed: apply_feed_field(state.feed, name, text)}
+
+ true ->
+ state
+ end
+
+ {:ok, state}
+ end
+
+ def handle_event(:end_document, _data, state), do: {:ok, state}
+
+ defp apply_entry_field(entry, "title", text), do: %{entry | title: text}
+ defp apply_entry_field(entry, "link", text), do: %{entry | url: text}
+ defp apply_entry_field(entry, "guid", text), do: %{entry | id: text}
+ defp apply_entry_field(entry, "description", text), do: %{entry | summary: text}
+ defp apply_entry_field(entry, "content:encoded", text), do: %{entry | content: text}
+ defp apply_entry_field(entry, "pubDate", text), do: %{entry | published_at: parse_date(text)}
+ defp apply_entry_field(entry, _name, _text), do: entry
+
+ defp apply_feed_field(feed, "title", text), do: %{feed | title: text}
+ defp apply_feed_field(feed, "description", text), do: %{feed | description: text}
+ defp apply_feed_field(feed, "link", text), do: %{feed | url: text}
+ defp apply_feed_field(feed, _name, _text), do: feed
+
+ defp finalize_entry(%Entry{id: nil, url: url} = entry) when is_binary(url) do
+ %{entry | id: url}
+ end
+
+ defp finalize_entry(entry), do: entry
+
+ defp text(parts) do
+ result =
+ parts
+ |> Enum.reverse()
+ |> IO.iodata_to_binary()
+ |> String.trim()
+
+ case result do
+ "" -> nil
+ trimmed -> trimmed
+ end
+ end
+
+ defp parse_date(nil), do: nil
+
+ defp parse_date(text) do
+ case DateTimeParser.parse_datetime(text) do
+ {:ok, datetime} ->
+ datetime
+
+ {:error, reason} ->
+ Logger.debug("Feeds.RSS unparseable pubDate #{inspect(text)}: #{inspect(reason)}")
+ nil
+ end
+ end
+end
diff --git a/lib/annot_at_web/components/layouts/root.html.heex b/lib/annot_at_web/components/layouts/root.html.heex
index 64e2aad..0b7592d 100644
--- a/lib/annot_at_web/components/layouts/root.html.heex
+++ b/lib/annot_at_web/components/layouts/root.html.heex
@@ -33,7 +33,6 @@
/>
-
Path.expand(__DIR__)
+ |> File.read!()
+
+ test "parses channel-level feed metadata" do
+ assert {:ok, %Feed{} = feed} = RSS.parse(@fixture)
+ assert "Sample Blog" == feed.title
+ assert "https://example.com" == feed.url
+ assert "Thoughts about things." == feed.description
+ end
+
+ test "parses entries with all fields" do
+ assert {:ok, %{entries: [first, _second]}} = RSS.parse(@fixture)
+
+ assert %Entry{} = first
+ assert "First Post" == first.title
+ assert "https://example.com/posts/first" == first.url
+ assert "abc" == first.id
+ assert "A short summary of the first post." == first.summary
+ assert "
The full content of the first post.
" == first.content
+ assert %DateTime{} = first.published_at
+ assert ~U[2024-10-02 13:00:00Z] == first.published_at
+ end
+
+ test "falls back to URL as ID when guid is missing" do
+ assert {:ok, %{entries: [_first, second]}} = RSS.parse(@fixture)
+ assert second.id == second.url
+ end
+
+ test "leaves published_at nil when pubDate is missing" do
+ assert {:ok, %{entries: [_first, second]}} = RSS.parse(@fixture)
+ refute second.published_at
+ end
+
+ test "leaves content nil when missing" do
+ assert {:ok, %{entries: [_first, second]}} = RSS.parse(@fixture)
+ refute second.content
+ end
+
+ test "returns an error on malformed" do
+ assert {:error, :invalid_feed} = RSS.parse("oops")
+ end
+
+ test "accepts a valid but empty blog" do
+ body = """
+
+
+
+ Brand New Blog
+ https://example.com
+ Nothing here yet.
+
+
+ """
+
+ assert {:ok, %Feed{title: "Brand New Blog", entries: []}} = RSS.parse(body)
+ end
+
+ test "rejects a feed with no channel title as invalid" do
+ body = """
+
+
+
+ https://example.com
+ No title here.
+
+
+ """
+
+ assert {:error, :invalid_feed} = RSS.parse(body)
+ end
+end
diff --git a/test/annot_at/feeds_test.exs b/test/annot_at/feeds_test.exs
new file mode 100644
index 0000000..7287f07
--- /dev/null
+++ b/test/annot_at/feeds_test.exs
@@ -0,0 +1,49 @@
+defmodule AnnotAt.FeedsTest do
+ use ExUnit.Case, async: true
+
+ alias AnnotAt.Feeds
+ alias AnnotAt.Feeds.Feed
+
+ @fixture "../support/fixtures/feeds/rss_sample.xml"
+ |> Path.expand(__DIR__)
+ |> File.read!()
+
+ describe "Feeds.parse/2" do
+ test "detects and dispatches RSS" do
+ assert {:ok, %Feed{title: "Sample Blog"}} = Feeds.parse(@fixture, "application/rss+xml")
+ end
+
+ test "rejects an unrecognized body" do
+ assert {:error, :unrecognized_feed} = Feeds.parse("not a feed at all", nil)
+ end
+ end
+
+ describe "Feeds.discover/2" do
+ test "finds and resolves a relative feed url" do
+ html = """
+
+
+
+
+
+ """
+
+ assert {:ok, "https://blog.example.com/feed.xml"} =
+ Feeds.discover(
+ html,
+ "https://blog.example.com"
+ )
+ end
+
+ test "returns :error when there's no feed link" do
+ html = """
+
+
+
+
+ """
+
+ assert :error = Feeds.discover(html, "https://blog.example.com")
+ end
+ end
+end
diff --git a/test/support/fixtures/feeds/rss_sample.xml b/test/support/fixtures/feeds/rss_sample.xml
new file mode 100644
index 0000000..b6c8613
--- /dev/null
+++ b/test/support/fixtures/feeds/rss_sample.xml
@@ -0,0 +1,27 @@
+
+
+
+ Sample Blog
+ https://example.com
+ Thoughts about things.
+
+ https://example.com/icon.png
+ Sample Blog Logo
+ https://example.com
+
+
+ First Post
+ https://example.com/posts/first
+ abc
+ A short summary of the first post.
+ The full content of the first post.
]]>
+ Wed, 02 Oct 2024 13:00:00 GMT
+
+
+ Second Post
+ https://example.com/posts/second
+ Another summary.
+
+
+