diff --git a/lib/annot_at/atproto/standard_site.ex b/lib/annot_at/atproto/standard_site.ex index 9e84624..85f241a 100644 --- a/lib/annot_at/atproto/standard_site.ex +++ b/lib/annot_at/atproto/standard_site.ex @@ -5,15 +5,17 @@ defmodule AnnotAt.Atproto.StandardSite do """ alias AnnotAt.Accounts + alias AnnotAt.Atproto.HTTP alias AnnotAt.Atproto.OAuth.Client alias AnnotAt.Atproto.StandardSite.Document alias AnnotAt.Atproto.StandardSite.Publication @publication "site.standard.publication" @document "site.standard.document" + @wellknown_path "/.well-known/site.standard.publication" @doc """ - Creates or updates the user's publication record (one per repo, rkey `self`). + Creates or updates the user's publication record. """ @spec put_publication(integer(), Publication.t()) :: {:ok, map()} | {:error, term()} def put_publication(user_id, %Publication{} = publication) do @@ -37,8 +39,80 @@ defmodule AnnotAt.Atproto.StandardSite do @doc """ The AT-URI of the user's publication record, used as a document's `site`. """ - @spec publication_uri(String.t()) :: String.t() - def publication_uri(did), do: "at://#{did}/#{@publication}/self" + @spec publication_uri(String.t(), String.t()) :: String.t() + def publication_uri(did, rkey), do: "at://#{did}/#{@publication}/#{rkey}" + + @doc """ + Verifies the website hosts the publication's AT-URI at its well-known path, + proving control of the domain. + """ + @spec verify_ownership(String.t(), String.t()) :: + :ok + | {:error, + :mismatch | :not_found | {:http_status, pos_integer()} | {:transport, term()}} + def verify_ownership(url, at_uri) do + url = wellknown_url(url) + + case HTTP.get_text(url) do + {:ok, body} -> + if String.trim(body) == at_uri do + :ok + else + {:error, :mismatch} + end + + {:error, {:http_status, 404}} -> + {:error, :not_found} + + {:error, _reason} = error -> + error + end + end + + @spec list_publications(integer()) :: + {:ok, [%{rkey: String.t(), url: String.t() | nil, name: String.t() | nil}]} + | {:error, :no_session} + def list_publications(user_id) do + with {:ok, user} <- fetch_user(user_id), + {:ok, %{"records" => records}} <- + Client.query(user.id, "com.atproto.repo.listRecords", + repo: user.did, + collection: @publication + ) do + {:ok, Enum.map(records, &to_existing/1)} + end + end + + @doc """ + Reads the user's publication document. + """ + @spec get_publication(integer(), String.t()) :: {:ok, map()} | {:error, term()} + def get_publication(user_id, rkey) do + with {:ok, user} <- fetch_user(user_id), + {:ok, %{"value" => value}} <- + Client.query(user.id, "com.atproto.repo.getRecord", + repo: user.did, + collection: @publication, + rkey: rkey + ) do + {:ok, value} + end + end + + @doc """ + Builds the publication document we create, from page metadata. + """ + @spec draft_publication(String.t(), %{title: String.t() | nil, description: String.t() | nil}) :: + map() + def draft_publication(url, %{title: title, description: description}) do + %{ + "$type" => @publication, + "name" => title, + "url" => url, + "description" => description, + "preferences" => %{"showInDiscover" => true} + } + end defp fetch_user(user_id) do case Accounts.get_user(user_id) do @@ -86,4 +160,24 @@ defmodule AnnotAt.Atproto.StandardSite do defp put_optional(map, _key, nil), do: map defp put_optional(map, key, value), do: Map.put(map, key, value) + + defp wellknown_url(url) do + uri = URI.parse(url) + + URI.to_string(%{ + uri + | path: @wellknown_path, + query: nil, + fragment: nil + }) + end + + defp to_existing(%{"uri" => uri, "value" => value}) do + rkey = + uri + |> String.split("/") + |> List.last() + + %{rkey: rkey, url: value["url"], name: value["name"]} + end end diff --git a/lib/annot_at/atproto/tid.ex b/lib/annot_at/atproto/tid.ex new file mode 100644 index 0000000..ba6f106 --- /dev/null +++ b/lib/annot_at/atproto/tid.ex @@ -0,0 +1,28 @@ +defmodule AnnotAt.Atproto.TID do + @moduledoc """ + atproto TID (timestamp identifier), frequenty used as record keys. + + https://atproto.com/specs/tid) + """ + + @alphabet "234567abcdefghijklmnopqrstuvwxyz" + + @spec now() :: String.t() + def now(time \\ System) do + # Shifts the time to make space for a random "clock ID" which basically + # just means we wouldn't get collisions if two TIDs were generated the + # same microsecond. Probably overkill. + clock_id = :rand.uniform(1024) - 1 + shifted = time.os_time(:microsecond) * 1024 + timestamp = shifted + clock_id + new(timestamp) + end + + @spec new(non_neg_integer()) :: String.t() + def new(int) do + int + |> Integer.digits(32) + |> Enum.map_join(&<<:binary.at(@alphabet, &1)>>) + |> String.pad_leading(13, "2") + end +end diff --git a/lib/annot_at/feeds.ex b/lib/annot_at/feeds.ex index eac0e41..69866d2 100644 --- a/lib/annot_at/feeds.ex +++ b/lib/annot_at/feeds.ex @@ -49,6 +49,26 @@ defmodule AnnotAt.Feeds do |> Enum.uniq_by(& &1.url) end + @doc """ + Extracts a page's title and description from its HTML. + """ + @spec metadata(binary()) :: %{title: String.t() | nil, description: String.t() | nil} + def metadata(html) when is_binary(html) do + doc = LazyHTML.from_document(html) + + %{ + title: + text_of(doc, "title") || + meta_content( + doc, + ~s(meta[property="og:title"]) + ), + description: + meta_content(doc, ~s(meta[name="description"])) || + meta_content(doc, ~s(meta[property="og:description"])) + } + end + defp source_from_attributes(attributes, base_url) do attributes = Map.new(attributes) @@ -84,4 +104,28 @@ defmodule AnnotAt.Feeds do true -> :unknown end end + + defp text_of(doc, selector) do + doc + |> LazyHTML.query(selector) + |> LazyHTML.text() + |> presence() + end + + defp meta_content(doc, selector) do + doc + |> LazyHTML.query(selector) + |> LazyHTML.attribute("content") + |> List.first() + |> presence() + end + + defp presence(nil), do: nil + + defp presence(str) do + case String.trim(str) do + "" -> nil + trimmed -> trimmed + end + end end diff --git a/lib/annot_at/feeds/client.ex b/lib/annot_at/feeds/client.ex new file mode 100644 index 0000000..1503165 --- /dev/null +++ b/lib/annot_at/feeds/client.ex @@ -0,0 +1,64 @@ +defmodule AnnotAt.Feeds.Client do + @moduledoc """ + Network stuff for feeds, like fetching pages or feeds and handing + them to the dicovery/parsing. + """ + + alias AnnotAt.Feeds + alias AnnotAt.Feeds.Feed + alias AnnotAt.Feeds.Source + + @receive_timeout 10_000 + + @doc """ + Fetches a page and returns the feeds discovered. + """ + @spec discover(String.t()) :: + {:ok, [Source.t()]} | {:error, {:http_status, pos_integer()} | {:transport, term()}} + def discover(url) do + with {:ok, html} <- get(url) do + {:ok, Feeds.discover(html, url)} + end + end + + @doc """ + Fetches and parses a single feed. + """ + @spec load(String.t()) :: + {:ok, Feed.t()} + | {:error, + :invalid_feed + | :unsupported_feed + | :unrecognized_feed + | {:http_status, pos_integer()} + | {:transport, term()}} + def load(feed_url) do + with {:ok, body} <- get(feed_url) do + Feeds.parse(body) + end + end + + @doc """ + Fetches a page and extracts its title/description. + """ + @spec metadata(String.t()) :: + {:ok, map()} | {:error, {:http_status, pos_integer()} | {:transport, term()}} + def metadata(url) do + with {:ok, html} <- get(url) do + {:ok, Feeds.metadata(html)} + end + end + + defp get(url) do + case Req.get(url, decode_body: false, receive_timeout: @receive_timeout) do + {:ok, %Req.Response{status: status, body: body}} when status in 200..299 -> + {:ok, body} + + {:ok, %Req.Response{status: status}} -> + {:error, {:http_status, status}} + + {:error, reason} -> + {:error, {:transport, reason}} + end + end +end diff --git a/lib/annot_at/publishing.ex b/lib/annot_at/publishing.ex index 7c1ca04..8b35f61 100644 --- a/lib/annot_at/publishing.ex +++ b/lib/annot_at/publishing.ex @@ -9,6 +9,7 @@ defmodule AnnotAt.Publishing do alias AnnotAt.Accounts.Scope alias AnnotAt.Accounts.User + alias AnnotAt.Atproto.TID alias AnnotAt.Publishing.Site alias AnnotAt.Repo @@ -21,14 +22,53 @@ defmodule AnnotAt.Publishing do Repo.get_by!(Site, id: id, user_id: user_id) end - def create_site(%Scope{user: %User{id: user_id}}, rkey, attrs) do - %Site{ - user_id: user_id, - rkey: rkey, + def create_site(%Scope{user: %User{id: user_id}}, url) do + case Repo.get_by(Site, user_id: user_id, url: url) do + nil -> + %Site{user_id: user_id} + |> Site.changeset(%{url: url}) + |> Repo.insert() + + %Site{} = site -> + {:ok, site} + end + end + + def use_new_publication(%Scope{user: %User{id: user_id}}, %Site{} = site) do + verify_user_ownership!(site, user_id) + rkey = TID.now() + + site + |> Ecto.Changeset.change(%{rkey: rkey}) + |> Repo.update() + end + + def use_existing_publication(%Scope{user: %User{id: user_id}}, %Site{} = site, rkey) do + verify_user_ownership!(site, user_id) + + site + |> Ecto.Changeset.change(%{rkey: rkey, published_at: DateTime.utc_now(:second)}) + |> Repo.update() + end + + def mark_verified(%Scope{user: %User{id: user_id}}, %Site{} = site) do + verify_user_ownership!(site, user_id) + + site + |> Ecto.Changeset.change(%{ verified_at: DateTime.utc_now(:second) - } - |> Site.changeset(attrs) - |> Repo.insert() + }) + |> Repo.update() + end + + def mark_published(%Scope{user: %User{id: user_id}}, %Site{} = site) do + verify_user_ownership!(site, user_id) + + site + |> Ecto.Changeset.change(%{ + published_at: DateTime.utc_now(:second) + }) + |> Repo.update() end def update_site(%Scope{user: %User{id: user_id}}, %Site{} = site, attrs) do diff --git a/lib/annot_at/publishing/site.ex b/lib/annot_at/publishing/site.ex index 45bb497..e5d5468 100644 --- a/lib/annot_at/publishing/site.ex +++ b/lib/annot_at/publishing/site.ex @@ -5,12 +5,13 @@ defmodule AnnotAt.Publishing.Site do @type t :: %__MODULE__{ user_id: integer(), - name: String.t(), + name: String.t() | nil, url: String.t(), description: String.t() | nil, - feed_url: String.t(), - rkey: String.t(), - verified_at: DateTime.t() + feed_url: String.t() | nil, + rkey: String.t() | nil, + verified_at: DateTime.t() | nil, + published_at: DateTime.t() | nil } schema "sites" do @@ -23,10 +24,11 @@ defmodule AnnotAt.Publishing.Site do # The url of the feed field :feed_url, :string # rkey of the site.standard.publication record - # deterministically generated from did and url field :rkey, :string # when the site was verified field :verified_at, :utc_datetime + # when the user clicked publish + field :published_at, :utc_datetime belongs_to :user, AnnotAt.Accounts.User @@ -35,14 +37,17 @@ defmodule AnnotAt.Publishing.Site do def changeset(site, attrs) do site - |> cast(attrs, [:name, :url, :description, :feed_url]) - |> validate_required([:name, :url, :feed_url, :rkey]) + |> cast(attrs, [:url, :name, :description, :feed_url]) + |> validate_required([:url]) + |> validate_length(:url, max: 2048) |> validate_length(:name, max: 255) |> validate_length(:description, max: 1000) - |> validate_length(:url, max: 2048) |> validate_length(:feed_url, max: 2048) - |> validate_length(:rkey, max: 512) + |> unique_constraint(:url, name: :sites_user_id_url_index) |> foreign_key_constraint(:user_id) - |> unique_constraint(:rkey, name: :sites_user_id_rkey_index) end + + def status(%__MODULE__{published_at: %DateTime{}}), do: :published + def status(%__MODULE__{verified_at: %DateTime{}}), do: :verified + def status(%__MODULE__{}), do: :draft end diff --git a/lib/annot_at/url.ex b/lib/annot_at/url.ex new file mode 100644 index 0000000..baf41bd --- /dev/null +++ b/lib/annot_at/url.ex @@ -0,0 +1,52 @@ +defmodule AnnotAt.URL do + @moduledoc """ + URL helpers. + """ + + @doc """ + Clean up URLs so we got consistent site "identifiers". + """ + def canonical(url) do + uri = + url + |> with_scheme() + |> URI.parse() + + URI.to_string(%{ + uri + | host: downcase(uri.host), + path: trim_slash(uri.path) + }) + end + + def valid?(url) do + uri = + url + |> String.trim() + |> canonical() + |> URI.parse() + + case uri do + %URI{host: host} when is_binary(host) -> + String.contains?(host, ".") + + _ -> + false + end + end + + defp with_scheme(url) do + if url =~ ~r{^https?://}i do + url + else + "https://" <> url + end + end + + defp downcase(nil), do: nil + defp downcase(host), do: String.downcase(host) + + defp trim_slash(nil), do: nil + defp trim_slash("/"), do: nil + defp trim_slash(path), do: String.trim_trailing(path, "/") +end diff --git a/lib/annot_at_web/components/core_components.ex b/lib/annot_at_web/components/core_components.ex index 8cc2a21..e62093c 100644 --- a/lib/annot_at_web/components/core_components.ex +++ b/lib/annot_at_web/components/core_components.ex @@ -453,6 +453,22 @@ defmodule AnnotAtWeb.CoreComponents do """ end + attr :label, :string, required: true + attr :value, :string, required: true + attr :sub, :string, required: true + attr :tint, :string, required: true + attr :shadow, :string, required: true + + def stat_card(assigns) do + ~H""" +
Connect a blog's RSS feed and we'll publish every new post to the ATmosphere.
- +We look for an RSS or Atom feed on your + site.
+Couldn't reach the + site.
+ + +No feed found on + this site.
+ +Reuse a publication you already have, + or create a new one.
+Prove you control this site by hosting + a small file.
+Host this file on your site, then + check again:
+
+ /.well-known/site.standard.publication
+
+ {@at_uri}
+ Couldn't read the + feed.
+ + +No posts in this feed yet.
+ <% latest -> %> +{length(feed.entries)} posts in the feed
+Couldn't read your + site.
+ + +Paste your blog's URL, we'll sniff out + its feed.
+