diff --git a/lib/mix/tasks/tempest.lexicon.generate.ex b/lib/mix/tasks/tempest.lexicon.generate.ex index 5061bd8..c8a2f45 100644 --- a/lib/mix/tasks/tempest.lexicon.generate.ex +++ b/lib/mix/tasks/tempest.lexicon.generate.ex @@ -15,6 +15,9 @@ defmodule Mix.Tasks.Tempest.Lexicon.Generate do * `--generated-at` - ISO8601 timestamp. Defaults to current UTC time. * `--include` - comma-separated document ids to include. Dependencies reached through local refs are included automatically. + * `--namespace` - comma-separated document id prefixes to include. This can + be used instead of enumerating every document id, for example + `--namespace app.bsky.feed,com.atproto.repo`. Operator update workflow: @@ -47,7 +50,8 @@ defmodule Mix.Tasks.Tempest.Lexicon.Generate do source_repo: :string, out: :string, generated_at: :string, - include: :string + include: :string, + namespace: :string ], aliases: [s: :source, c: :commit, o: :out] ) @@ -63,7 +67,7 @@ defmodule Mix.Tasks.Tempest.Lexicon.Generate do generated_at = Keyword.get(opts, :generated_at, generated_at()) with {:ok, documents, _manifest} <- LocalProvider.load(paths: [source]), - documents = select_documents(documents, Keyword.get(opts, :include)), + documents = select_documents(documents, Keyword.get(opts, :include), Keyword.get(opts, :namespace)), :ok <- Document.validate_documents(documents) do File.mkdir_p!(Path.dirname(output)) File.write!(output, module_source(documents, source_repo, commit, generated_at)) @@ -73,12 +77,14 @@ defmodule Mix.Tasks.Tempest.Lexicon.Generate do end end - defp select_documents(documents, nil), do: sort_documents(documents) + defp select_documents(documents, nil, nil), do: sort_documents(documents) - defp select_documents(documents, include) do + defp select_documents(documents, include, namespace) do ids = - include - |> String.split(",", trim: true) + documents + |> included_document_ids(include) + |> namespace_document_ids(namespace) + |> Enum.uniq() by_id = Map.new(documents, &{Map.fetch!(&1, "id"), &1}) @@ -89,6 +95,27 @@ defmodule Mix.Tasks.Tempest.Lexicon.Generate do |> sort_documents() end + defp included_document_ids(documents, nil), do: Enum.map(documents, &Map.fetch!(&1, "id")) + + defp included_document_ids(_documents, include) do + include + |> String.split(",", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + end + + defp namespace_document_ids(ids, nil), do: ids + + defp namespace_document_ids(ids, namespace) do + namespaces = + namespace + |> String.split(",", trim: true) + |> Enum.map(&String.trim/1) + |> Enum.reject(&(&1 == "")) + + Enum.filter(ids, fn id -> Enum.any?(namespaces, &(id == &1 or String.starts_with?(id, &1 <> "."))) end) + end + defp expand_dependencies(ids, by_id, seen) do ids |> Enum.reduce(seen, fn id, seen -> diff --git a/lib/mix/tasks/tempest.lexicon.sync.ex b/lib/mix/tasks/tempest.lexicon.sync.ex index 0e954e2..89be9ac 100644 --- a/lib/mix/tasks/tempest.lexicon.sync.ex +++ b/lib/mix/tasks/tempest.lexicon.sync.ex @@ -44,6 +44,7 @@ defmodule Mix.Tasks.Tempest.Lexicon.Sync do app/bsky/embed/recordWithMedia.json app/bsky/embed/video.json app/bsky/feed/defs.json + app/bsky/feed/post.json app/bsky/feed/postgate.json app/bsky/feed/threadgate.json app/bsky/graph/defs.json diff --git a/lib/tempest/lexicon/bundled.ex b/lib/tempest/lexicon/bundled.ex index e09f1b6..337bf9c 100644 --- a/lib/tempest/lexicon/bundled.ex +++ b/lib/tempest/lexicon/bundled.ex @@ -10,7 +10,7 @@ defmodule Tempest.Lexicon.Bundled do @behaviour Tempest.Lexicon.Provider @manifest %{ - "document_count" => 71, + "document_count" => 72, "document_ids" => [ "app.bsky.actor.defs", "app.bsky.actor.getPreferences", @@ -24,6 +24,7 @@ defmodule Tempest.Lexicon.Bundled do "app.bsky.embed.recordWithMedia", "app.bsky.embed.video", "app.bsky.feed.defs", + "app.bsky.feed.post", "app.bsky.feed.postgate", "app.bsky.feed.threadgate", "app.bsky.graph.defs", @@ -1746,6 +1747,108 @@ defmodule Tempest.Lexicon.Bundled do "id" => "app.bsky.feed.defs", "lexicon" => 1 }, + %{ + "defs" => %{ + "entity" => %{ + "description" => "Deprecated: use facets instead.", + "properties" => %{ + "index" => %{"ref" => "#textSlice", "type" => "ref"}, + "type" => %{ + "description" => "Expected values are 'mention' and 'link'.", + "type" => "string" + }, + "value" => %{"type" => "string"} + }, + "required" => ["index", "type", "value"], + "type" => "object" + }, + "main" => %{ + "description" => "Record containing a Bluesky post.", + "key" => "tid", + "record" => %{ + "properties" => %{ + "createdAt" => %{ + "description" => "Client-declared timestamp when this post was originally created.", + "format" => "datetime", + "type" => "string" + }, + "embed" => %{ + "refs" => [ + "app.bsky.embed.images", + "app.bsky.embed.video", + "app.bsky.embed.gallery", + "app.bsky.embed.external", + "app.bsky.embed.record", + "app.bsky.embed.recordWithMedia" + ], + "type" => "union" + }, + "entities" => %{ + "description" => "DEPRECATED: replaced by app.bsky.richtext.facet.", + "items" => %{"ref" => "#entity", "type" => "ref"}, + "type" => "array" + }, + "facets" => %{ + "description" => "Annotations of text (mentions, URLs, hashtags, etc)", + "items" => %{"ref" => "app.bsky.richtext.facet", "type" => "ref"}, + "type" => "array" + }, + "labels" => %{ + "description" => "Self-label values for this post. Effectively content warnings.", + "refs" => ["com.atproto.label.defs#selfLabels"], + "type" => "union" + }, + "langs" => %{ + "description" => "Indicates human language of post primary text content.", + "items" => %{"format" => "language", "type" => "string"}, + "maxLength" => 3, + "type" => "array" + }, + "reply" => %{"ref" => "#replyRef", "type" => "ref"}, + "tags" => %{ + "description" => "Additional hashtags, in addition to any included in post text and facets.", + "items" => %{ + "maxGraphemes" => 64, + "maxLength" => 640, + "type" => "string" + }, + "maxLength" => 8, + "type" => "array" + }, + "text" => %{ + "description" => "The primary post content. May be an empty string, if there are embeds.", + "maxGraphemes" => 300, + "maxLength" => 3000, + "type" => "string" + } + }, + "required" => ["text", "createdAt"], + "type" => "object" + }, + "type" => "record" + }, + "replyRef" => %{ + "properties" => %{ + "parent" => %{"ref" => "com.atproto.repo.strongRef", "type" => "ref"}, + "root" => %{"ref" => "com.atproto.repo.strongRef", "type" => "ref"} + }, + "required" => ["root", "parent"], + "type" => "object" + }, + "textSlice" => %{ + "description" => + "Deprecated. Use app.bsky.richtext instead -- A text segment. Start is inclusive, end is exclusive. Indices are for utf16-encoded strings.", + "properties" => %{ + "end" => %{"minimum" => 0, "type" => "integer"}, + "start" => %{"minimum" => 0, "type" => "integer"} + }, + "required" => ["start", "end"], + "type" => "object" + } + }, + "id" => "app.bsky.feed.post", + "lexicon" => 1 + }, %{ "defs" => %{ "disableRule" => %{ diff --git a/lib/tempest/lexicon/local_provider.ex b/lib/tempest/lexicon/local_provider.ex index 524c42a..addce51 100644 --- a/lib/tempest/lexicon/local_provider.ex +++ b/lib/tempest/lexicon/local_provider.ex @@ -13,18 +13,18 @@ defmodule Tempest.Lexicon.LocalProvider do @impl true def load(opts) do paths = Keyword.get(opts, :paths, []) + repositories = Keyword.get(opts, :repositories, []) limits = Keyword.merge(@default_limits, opts) - with {:ok, files} <- expand_files(paths, limits), + with {:ok, files} <- expand_files(paths, repositories, limits), {:ok, documents} <- read_documents(files, limits) do {:ok, documents, local_manifest(files, documents)} end end - defp expand_files(paths, limits) when is_list(paths) do + defp expand_files(paths, repositories, limits) when is_list(paths) and is_list(repositories) do files = - paths - |> Enum.flat_map(&lexicon_files/1) + (Enum.flat_map(paths, &lexicon_files/1) ++ Enum.flat_map(repositories, &repository_files/1)) |> Enum.uniq() |> Enum.sort() @@ -35,7 +35,7 @@ defmodule Tempest.Lexicon.LocalProvider do end end - defp expand_files(_paths, _limits), do: {:error, :invalid_lexicon_paths} + defp expand_files(_paths, _repositories, _limits), do: {:error, :invalid_lexicon_paths} defp read_documents(files, limits) do Enum.reduce_while(files, {:ok, []}, fn file, {:ok, documents} -> @@ -81,6 +81,43 @@ defmodule Tempest.Lexicon.LocalProvider do defp lexicon_files(_path), do: [] + defp repository_files(repository) when is_list(repository) or is_map(repository) do + repository = Map.new(repository) + root = Map.get(repository, :path) || Map.get(repository, "path") + namespaces = Map.get(repository, :namespaces) || Map.get(repository, "namespaces") || [] + + with true <- is_binary(root), + lexicon_root when is_binary(lexicon_root) <- lexicon_root(root), + true <- is_list(namespaces) do + namespaces + |> Enum.flat_map(&namespace_files(lexicon_root, &1)) + else + _invalid -> [] + end + end + + defp repository_files(_repository), do: [] + + defp lexicon_root(root) do + cond do + File.dir?(Path.join(root, "lexicons")) -> Path.join(root, "lexicons") + File.dir?(root) -> root + true -> nil + end + end + + defp namespace_files(root, namespace) when is_binary(namespace) do + namespace_path = Path.join([root | String.split(namespace, ".", trim: true)]) + + cond do + File.dir?(namespace_path) -> Path.wildcard(Path.join(namespace_path, "**/*.json")) + File.regular?(namespace_path <> ".json") -> [namespace_path <> ".json"] + true -> [] + end + end + + defp namespace_files(_root, _namespace), do: [] + defp local_manifest(files, documents) do %{ "source" => "local", diff --git a/lib/tempest/lexicon/registry.ex b/lib/tempest/lexicon/registry.ex index a9685b0..3234c4e 100644 --- a/lib/tempest/lexicon/registry.ex +++ b/lib/tempest/lexicon/registry.ex @@ -27,6 +27,7 @@ defmodule Tempest.Lexicon.Registry do bundled_provider: Tempest.Lexicon.Bundled, documents: [], paths: [], + repositories: [], limits: [], external_resolver: @default_external_resolver ] @@ -142,7 +143,10 @@ defmodule Tempest.Lexicon.Registry do {:ok, configured_documents} <- normalize_documents(Keyword.get(config, :documents, [])), {:ok, local_documents, local_manifest} <- Tempest.Lexicon.LocalProvider.load( - Keyword.merge(Keyword.get(config, :limits, []), paths: Keyword.get(config, :paths, [])) + Keyword.merge(Keyword.get(config, :limits, []), + paths: Keyword.get(config, :paths, []), + repositories: Keyword.get(config, :repositories, []) + ) ) do documents = bundled_documents ++ configured_documents ++ local_documents diff --git a/priv/lexicons/official/app/bsky/feed/post.json b/priv/lexicons/official/app/bsky/feed/post.json new file mode 100644 index 0000000..a4b98ef --- /dev/null +++ b/priv/lexicons/official/app/bsky/feed/post.json @@ -0,0 +1,97 @@ +{ + "lexicon": 1, + "id": "app.bsky.feed.post", + "defs": { + "main": { + "type": "record", + "description": "Record containing a Bluesky post.", + "key": "tid", + "record": { + "type": "object", + "required": ["text", "createdAt"], + "properties": { + "text": { + "type": "string", + "maxLength": 3000, + "maxGraphemes": 300, + "description": "The primary post content. May be an empty string, if there are embeds." + }, + "entities": { + "type": "array", + "description": "DEPRECATED: replaced by app.bsky.richtext.facet.", + "items": { "type": "ref", "ref": "#entity" } + }, + "facets": { + "type": "array", + "description": "Annotations of text (mentions, URLs, hashtags, etc)", + "items": { "type": "ref", "ref": "app.bsky.richtext.facet" } + }, + "reply": { "type": "ref", "ref": "#replyRef" }, + "embed": { + "type": "union", + "refs": [ + "app.bsky.embed.images", + "app.bsky.embed.video", + "app.bsky.embed.gallery", + "app.bsky.embed.external", + "app.bsky.embed.record", + "app.bsky.embed.recordWithMedia" + ] + }, + "langs": { + "type": "array", + "description": "Indicates human language of post primary text content.", + "maxLength": 3, + "items": { "type": "string", "format": "language" } + }, + "labels": { + "type": "union", + "description": "Self-label values for this post. Effectively content warnings.", + "refs": ["com.atproto.label.defs#selfLabels"] + }, + "tags": { + "type": "array", + "description": "Additional hashtags, in addition to any included in post text and facets.", + "maxLength": 8, + "items": { "type": "string", "maxLength": 640, "maxGraphemes": 64 } + }, + "createdAt": { + "type": "string", + "format": "datetime", + "description": "Client-declared timestamp when this post was originally created." + } + } + } + }, + "replyRef": { + "type": "object", + "required": ["root", "parent"], + "properties": { + "root": { "type": "ref", "ref": "com.atproto.repo.strongRef" }, + "parent": { "type": "ref", "ref": "com.atproto.repo.strongRef" } + } + }, + "entity": { + "type": "object", + "description": "Deprecated: use facets instead.", + "required": ["index", "type", "value"], + "properties": { + "index": { "type": "ref", "ref": "#textSlice" }, + "type": { + "type": "string", + "description": "Expected values are 'mention' and 'link'." + }, + "value": { "type": "string" } + } + }, + "textSlice": { + "type": "object", + "description": "Deprecated. Use app.bsky.richtext instead -- A text segment. Start is inclusive, end is exclusive. Indices are for utf16-encoded strings.", + "required": ["start", "end"], + "properties": { + "start": { "type": "integer", "minimum": 0 }, + "end": { "type": "integer", "minimum": 0 } + } + } + } +} diff --git a/priv/lexicons/smoke/app/bsky/feed/post.json b/priv/lexicons/smoke/app/bsky/feed/post.json new file mode 100644 index 0000000..09c71fc --- /dev/null +++ b/priv/lexicons/smoke/app/bsky/feed/post.json @@ -0,0 +1,25 @@ +{ + "lexicon": 1, + "id": "app.bsky.feed.post", + "defs": { + "main": { + "type": "record", + "key": "tid", + "record": { + "type": "object", + "required": ["text", "createdAt"], + "properties": { + "text": { + "type": "string", + "maxLength": 3000, + "maxGraphemes": 300 + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} diff --git a/test/tempest/lexicon/bundled_manifest_test.exs b/test/tempest/lexicon/bundled_manifest_test.exs index 0474326..9fcd1e0 100644 --- a/test/tempest/lexicon/bundled_manifest_test.exs +++ b/test/tempest/lexicon/bundled_manifest_test.exs @@ -10,6 +10,7 @@ defmodule Tempest.Lexicon.BundledManifestTest do assert manifest["source_repo"] == "https://github.com/bluesky-social/atproto" assert manifest["source_commit"] =~ ~r/^[0-9a-f]{40}$/ assert manifest["document_count"] == length(Bundled.documents()) + assert "app.bsky.feed.post" in manifest["document_ids"] assert "com.atproto.repo.applyWrites" in manifest["document_ids"] assert "com.atproto.sync.getBlocks" in manifest["document_ids"] assert "com.atproto.sync.requestCrawl" in manifest["document_ids"] diff --git a/test/tempest/lexicon/com_atproto_compat_test.exs b/test/tempest/lexicon/com_atproto_compat_test.exs index 485d7e3..0bcb365 100644 --- a/test/tempest/lexicon/com_atproto_compat_test.exs +++ b/test/tempest/lexicon/com_atproto_compat_test.exs @@ -157,7 +157,7 @@ defmodule Tempest.Lexicon.OfficialComAtprotoCompatibilityTest do assert :ok = Document.validate_documents(@official_com_atproto_subset) end - test "compatibility target excludes official app.bsky profile post and follow records" do + test "com.atproto compatibility subset stays scoped to com.atproto documents" do ids = Enum.map(@official_com_atproto_subset, &Map.fetch!(&1, "id")) assert Enum.all?(ids, &String.starts_with?(&1, "com.atproto.")) diff --git a/test/tempest/lexicon/generate_task_test.exs b/test/tempest/lexicon/generate_task_test.exs index 68bbf17..8ec990c 100644 --- a/test/tempest/lexicon/generate_task_test.exs +++ b/test/tempest/lexicon/generate_task_test.exs @@ -57,6 +57,59 @@ defmodule Tempest.Lexicon.GenerateTaskTest do assert generated =~ ~s("example.app.subject") end + test "generates bundled module from namespace prefixes with dependencies" do + source = tmp_dir!("namespace-source") + output = Path.join(tmp_dir!("namespace-output"), "bundled.ex") + + write_json!( + Path.join(source, "example.app.note.json"), + lexicon("example.app.note", %{ + "main" => %{ + "type" => "record", + "key" => "any", + "record" => %{ + "type" => "object", + "properties" => %{"subject" => %{"type" => "ref", "ref" => "com.example.subject"}} + } + } + }) + ) + + write_json!( + Path.join(source, "example.app.profile.json"), + lexicon("example.app.profile", %{"main" => %{"type" => "object"}}) + ) + + write_json!( + Path.join(source, "com.example.subject.json"), + lexicon("com.example.subject", %{"main" => %{"type" => "object"}}) + ) + + write_json!( + Path.join(source, "other.app.note.json"), + lexicon("other.app.note", %{"main" => %{"type" => "object"}}) + ) + + Generate.run([ + "--source", + source, + "--commit", + "abc123", + "--namespace", + "example.app", + "--out", + output + ]) + + generated = File.read!(output) + + assert generated =~ ~s("document_count" => 3) + assert generated =~ ~s("example.app.note") + assert generated =~ ~s("example.app.profile") + assert generated =~ ~s("com.example.subject") + refute generated =~ ~s("other.app.note") + end + defp lexicon(id, defs), do: %{"lexicon" => 1, "id" => id, "defs" => defs} defp tmp_dir!(name) do @@ -65,5 +118,8 @@ defmodule Tempest.Lexicon.GenerateTaskTest do path end - defp write_json!(path, data), do: File.write!(path, Jason.encode!(data)) + defp write_json!(path, data) do + File.mkdir_p!(Path.dirname(path)) + File.write!(path, Jason.encode!(data)) + end end diff --git a/test/tempest/lexicon/registry_test.exs b/test/tempest/lexicon/registry_test.exs index 6ea3251..4cd0206 100644 --- a/test/tempest/lexicon/registry_test.exs +++ b/test/tempest/lexicon/registry_test.exs @@ -46,6 +46,37 @@ defmodule Tempest.Lexicon.RegistryTest do Registry.fetch_definition("com.atproto.repo.strongRef") end + test "loads repository namespaces from a lexicons directory" do + directory = tmp_dir!("namespace-lexicons") + write_json!(Path.join([directory, "example", "app", "post.json"]), custom_record("example.app.post")) + write_json!(Path.join([directory, "example", "app", "like.json"]), custom_record("example.app.like")) + write_json!(Path.join([directory, "other", "app", "post.json"]), custom_record("other.app.post")) + + Application.put_env(:tempest, Registry, + bundled?: false, + repositories: [[path: directory, namespaces: ["example.app"]]] + ) + + assert :ok = Registry.validate_startup!() + assert {:ok, _document, %{"type" => "record"}} = Registry.fetch_record("example.app.post") + assert {:ok, _document, %{"type" => "record"}} = Registry.fetch_record("example.app.like") + assert {:error, :unknown_lexicon} = Registry.fetch_record("other.app.post") + end + + test "loads repository namespaces from a git checkout root" do + checkout = tmp_dir!("namespace-repo") + lexicons = Path.join(checkout, "lexicons") + write_json!(Path.join([lexicons, "example", "feed", "post.json"]), custom_record("example.feed.post")) + + Application.put_env(:tempest, Registry, + bundled?: false, + repositories: [[path: checkout, namespaces: ["example.feed"]]] + ) + + assert :ok = Registry.validate_startup!() + assert {:ok, _document, %{"type" => "record"}} = Registry.fetch_record("example.feed.post") + end + test "configured local documents are validated and can add custom record schemas" do directory = tmp_dir!("custom-lexicons") @@ -232,7 +263,10 @@ defmodule Tempest.Lexicon.RegistryTest do path end - defp write_json!(path, data), do: File.write!(path, Jason.encode!(data)) + defp write_json!(path, data) do + File.mkdir_p!(Path.dirname(path)) + File.write!(path, Jason.encode!(data)) + end end defmodule Tempest.Lexicon.RegistryTest.Resolver do diff --git a/test/tempest_web/xrpc/records_test.exs b/test/tempest_web/xrpc/records_test.exs index fa7c026..6365ce2 100644 --- a/test/tempest_web/xrpc/records_test.exs +++ b/test/tempest_web/xrpc/records_test.exs @@ -161,7 +161,7 @@ defmodule TempestWeb.Xrpc.RecordsTest do assert %{"error" => "InvalidRequest"} = json_response(denied_conn, 400) end - test "createRecord persists Bluesky post-shaped records with unknown validation status", %{conn: conn} do + test "createRecord validates and persists Bluesky post records", %{conn: conn} do account = create_account!(conn, "records-bsky-post.test", "records-bsky-post@example.com") created = @@ -170,7 +170,8 @@ defmodule TempestWeb.Xrpc.RecordsTest do |> post(~p"/xrpc/com.atproto.repo.createRecord", %{ "repo" => account["did"], "collection" => "app.bsky.feed.post", - "rkey" => "3kpost", + "rkey" => "3mo7hac7efyou", + "validate" => true, "record" => %{ "$type" => "app.bsky.feed.post", "text" => "hello from bsky-shaped post", @@ -179,8 +180,8 @@ defmodule TempestWeb.Xrpc.RecordsTest do }) |> json_response(200) - assert created["uri"] == "at://#{account["did"]}/app.bsky.feed.post/3kpost" - assert created["validationStatus"] == "unknown" + assert created["uri"] == "at://#{account["did"]}/app.bsky.feed.post/3mo7hac7efyou" + assert created["validationStatus"] == "valid" assert scalar(repo_db(account["did"]), "SELECT COUNT(*) FROM records") == 1 response = @@ -189,7 +190,7 @@ defmodule TempestWeb.Xrpc.RecordsTest do |> get(~p"/xrpc/com.atproto.repo.getRecord", %{ "repo" => account["did"], "collection" => "app.bsky.feed.post", - "rkey" => "3kpost" + "rkey" => "3mo7hac7efyou" }) |> json_response(200)