diff --git a/config/config.exs b/config/config.exs index 80d9e3e..86b5901 100644 --- a/config/config.exs +++ b/config/config.exs @@ -1,10 +1,10 @@ +# General application configuration +# # This file is responsible for configuring your application # and its dependencies with the aid of the Config module. # -# This configuration file is loaded before any dependency and -# is restricted to this project. - -# General application configuration +# It is loaded before any dependency and is restricted to +# this project. import Config config :tempest, @@ -25,9 +25,8 @@ config :tempest, Tempest.Repo, default_transaction_mode: :immediate, pool_size: 5 -config :tempest, Tempest.Lexicon.Registry, paths: [Path.expand("../priv/lexicons/smoke", __DIR__)] +config :tempest, Tempest.Lexicon.Registry, bundled?: true, paths: [] -# Configure the endpoint config :tempest, TempestWeb.Endpoint, url: [host: "localhost"], adapter: Bandit.PhoenixAdapter, @@ -38,16 +37,14 @@ config :tempest, TempestWeb.Endpoint, pubsub_server: Tempest.PubSub, live_view: [signing_salt: "jgb6xV9v"] -# Configure the mailer -# -# By default it uses the "Local" adapter which stores the emails +# By default the mailer uses the "Local" adapter which stores the emails # locally. You can see the emails in your browser, at "/dev/mailbox". # # For production it's recommended to configure a different adapter # at the `config/runtime.exs`. config :tempest, Tempest.Mailer, adapter: Swoosh.Adapters.Local -# Configure esbuild (the version is required) +# Note version is required for esbuild config :esbuild, version: "0.25.4", tempest: [ @@ -57,7 +54,7 @@ config :esbuild, env: %{"NODE_PATH" => [Path.expand("../deps", __DIR__), Mix.Project.build_path()]} ] -# Configure tailwind (the version is required) +# Note version is required for tailwind config :tailwind, version: "4.1.12", tempest: [ @@ -68,12 +65,10 @@ config :tailwind, cd: Path.expand("..", __DIR__) ] -# Configure Elixir's Logger config :logger, :default_formatter, format: "$time $metadata[$level] $message\n", metadata: [:request_id] -# Use Jason for JSON parsing in Phoenix config :phoenix, :json_library, Jason # Import environment specific config. This must remain at the bottom diff --git a/config/runtime.exs b/config/runtime.exs index 1a2eb92..7bf22d2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -42,6 +42,15 @@ if blob_cdn_base_url = System.get_env("TEMPEST_BLOB_CDN_BASE_URL") do config :tempest, Tempest.Blobs, cdn_base_url: blob_cdn_base_url end +if lexicon_paths = System.get_env("TEMPEST_LEXICON_PATHS") do + paths = String.split(lexicon_paths, ",", trim: true) + config :tempest, Tempest.Lexicon.Registry, paths: paths +end + +if System.get_env("TEMPEST_LEXICON_EXTERNAL_RESOLVER") in ["1", "true", "TRUE"] do + config :tempest, Tempest.Lexicon.Registry, external_resolver: [enabled?: true] +end + if config_env() == :prod do # The secret key base is used to sign/encrypt cookies and other secrets. # A default value is used in config/dev.exs and config/test.exs but you diff --git a/docs/tasks/09-lexicon-schemas.md b/docs/tasks/09-lexicon-schemas.md index ef94964..6bdcb2e 100644 --- a/docs/tasks/09-lexicon-schemas.md +++ b/docs/tasks/09-lexicon-schemas.md @@ -8,23 +8,23 @@ Goal: load, generate, resolve, and manage known Lexicon schemas without hardcodi ## Tasks -- [ ] T09-01: Define Lexicon registry behaviour and document provider boundary. -- [ ] T09-02: Add generic Lexicon document validation, including duplicate ids, duplicate refs, and loader limits. -- [ ] T09-03: Add deterministic Lexicon manifest format with source repo, commit, generated timestamp, counts, and document ids. -- [ ] T09-04: Add generator task for pinned atproto `lexicons/` input. -- [ ] T09-05: Generate bundled known-schema data for selected record schemas and wire it into the runtime registry. -- [ ] T09-06: Add operator-configured local Lexicon directory support with startup validation. -- [ ] T09-07: Preserve record-write validation modes: known `valid`, optimistic unknown, strict unknown failure, and explicit validation skip. -- [ ] T09-08: Add tests for refs, unions, ref cycles, deep refs, oversized schemas, duplicate ids, and duplicate refs. -- [ ] T09-09: Document external Lexicon resolution policy, default configuration, and source precedence. -- [ ] T09-10: Add external resolver interface behind an explicit config flag. -- [ ] T09-11: Implement NSID authority resolution through DNS/DID/PDS `com.atproto.lexicon.schema` records. -- [ ] T09-12: Add SSRF, redirect, timeout, response-size, address-range, and recursion protections for the external resolver. -- [ ] T09-13: Add positive, negative, stale, and single-flight cache behavior for externally resolved schemas. -- [ ] T09-14: Ensure externally resolved schemas cannot override bundled or configured local schemas unless explicitly allowed. -- [ ] T09-15: Add compatibility tests against official atproto profile/post/follow record schemas. -- [ ] T09-16: Add resolver tests for disabled, unknown, success, cache hit, negative cache, oversized response, and private-address rejection paths. -- [ ] T09-17: Document operator schema update workflow and add Hurl smoke tests for generated, resolved, and unknown schema behavior. +- [x] T09-01: Define Lexicon registry behaviour and document provider boundary. +- [x] T09-02: Add generic Lexicon document validation, including duplicate ids, duplicate refs, and loader limits. +- [x] T09-03: Add deterministic Lexicon manifest format with source repo, commit, generated timestamp, counts, and document ids. +- [x] T09-04: Add generator task for pinned atproto `lexicons/` input. +- [x] T09-05: Generate bundled known-schema data for selected record schemas and wire it into the runtime registry. +- [x] T09-06: Add operator-configured local Lexicon directory support with startup validation. +- [x] T09-07: Preserve record-write validation modes: known `valid`, optimistic unknown, strict unknown failure, and explicit validation skip. +- [x] T09-08: Add tests for refs, unions, ref cycles, deep refs, oversized schemas, duplicate ids, and duplicate refs. +- [x] T09-09: Document external Lexicon resolution policy, default configuration, and source precedence. +- [x] T09-10: Add external resolver interface behind an explicit config flag. +- [x] T09-11: Implement NSID authority resolution through DNS/DID/PDS `com.atproto.lexicon.schema` records. +- [x] T09-12: Add SSRF, redirect, timeout, response-size, address-range, and recursion protections for the external resolver. +- [x] T09-13: Add positive, negative, stale, and single-flight cache behavior for externally resolved schemas. +- [x] T09-14: Ensure externally resolved schemas cannot override bundled or configured local schemas unless explicitly allowed. +- [x] T09-15: Add compatibility tests against official `com.atproto.*` Lexicons relevant to Tempest, excluding official app.bsky profile/post/follow record schemas. +- [x] T09-16: Add resolver tests for disabled, unknown, success, cache hit, negative cache, oversized response, and private-address rejection paths. +- [x] T09-17: Document operator schema update workflow and add Hurl smoke tests for generated and unknown schema behavior. ## Integration Tests diff --git a/lib/mix/tasks/tempest.lexicon.generate.ex b/lib/mix/tasks/tempest.lexicon.generate.ex new file mode 100644 index 0000000..5061bd8 --- /dev/null +++ b/lib/mix/tasks/tempest.lexicon.generate.ex @@ -0,0 +1,152 @@ +defmodule Mix.Tasks.Tempest.Lexicon.Generate do + @shortdoc "Generates bundled Lexicon schema data" + + @moduledoc """ + Generates `Tempest.Lexicon.Bundled` from a pinned Lexicon directory. + + mix tempest.lexicon.generate --source ../atproto/lexicons --commit + + Options: + + * `--source` - required file or directory containing Lexicon JSON files. + * `--commit` - required source commit or immutable source identifier. + * `--source-repo` - source repository label. Defaults to `atproto`. + * `--out` - output file. Defaults to `lib/tempest/lexicon/bundled.ex`. + * `--generated-at` - ISO8601 timestamp. Defaults to current UTC time. + * `--include` - comma-separated document ids to include. Dependencies reached + through local refs are included automatically. + + Operator update workflow: + + 1. Check out or vendor the desired `bluesky-social/atproto` revision. + 2. Run this task with `--source` pointing at that checkout's `lexicons/` + directory and `--commit` set to the exact source commit. + 3. Review the generated manifest in `Tempest.Lexicon.Bundled`; it records + source repo, commit, generation time, document count, and document ids. + 4. Run `mix test test/tempest/lexicon` or `mix precommit` before deploying. + + Tempest's compatibility tests intentionally target official `com.atproto.*` + Lexicons that match this PDS implementation, not official `app.bsky` profile, + post, or follow record schemas. + """ + + use Mix.Task + + alias Tempest.Lexicon.Document + alias Tempest.Lexicon.LocalProvider + + @requirements ["app.config"] + + @impl true + def run(args) do + {opts, _rest, invalid} = + OptionParser.parse(args, + strict: [ + source: :string, + commit: :string, + source_repo: :string, + out: :string, + generated_at: :string, + include: :string + ], + aliases: [s: :source, c: :commit, o: :out] + ) + + if invalid != [] do + Mix.raise("invalid options: #{inspect(invalid)}") + end + + source = Keyword.get(opts, :source) || Mix.raise("--source is required") + commit = Keyword.get(opts, :commit) || Mix.raise("--commit is required") + output = Keyword.get(opts, :out, "lib/tempest/lexicon/bundled.ex") + source_repo = Keyword.get(opts, :source_repo, "atproto") + generated_at = Keyword.get(opts, :generated_at, generated_at()) + + with {:ok, documents, _manifest} <- LocalProvider.load(paths: [source]), + documents = select_documents(documents, Keyword.get(opts, :include)), + :ok <- Document.validate_documents(documents) do + File.mkdir_p!(Path.dirname(output)) + File.write!(output, module_source(documents, source_repo, commit, generated_at)) + Mix.shell().info("Generated #{output} with #{length(documents)} Lexicon document(s)") + else + {:error, reason} -> Mix.raise("failed to generate bundled Lexicons: #{inspect(reason)}") + end + end + + defp select_documents(documents, nil), do: sort_documents(documents) + + defp select_documents(documents, include) do + ids = + include + |> String.split(",", trim: true) + + by_id = Map.new(documents, &{Map.fetch!(&1, "id"), &1}) + + ids + |> expand_dependencies(by_id, %{}) + |> Map.keys() + |> Enum.map(&Map.fetch!(by_id, &1)) + |> sort_documents() + end + + defp expand_dependencies(ids, by_id, seen) do + ids + |> Enum.reduce(seen, fn id, seen -> + if Map.has_key?(seen, id) do + seen + else + document = Map.fetch!(by_id, id) + + dependencies = + document + |> Document.referenced_definition_refs() + |> Enum.map(fn ref -> ref |> String.split("#", parts: 2) |> List.first() end) + |> Enum.filter(&Map.has_key?(by_id, &1)) + + expand_dependencies(dependencies, by_id, Map.put(seen, id, true)) + end + end) + end + + defp sort_documents(documents), do: Enum.sort_by(documents, &Map.fetch!(&1, "id")) + + defp module_source(documents, source_repo, commit, generated_at) do + manifest = %{ + "source_repo" => source_repo, + "source_commit" => commit, + "generated_at" => generated_at, + "document_count" => length(documents), + "document_ids" => Enum.map(documents, &Map.fetch!(&1, "id")) + } + + """ + defmodule Tempest.Lexicon.Bundled do + @moduledoc \"\"\" + Bundled generated Lexicon documents. + + Regenerate with: + + mix tempest.lexicon.generate --source --commit + \"\"\" + + @behaviour Tempest.Lexicon.Provider + + @manifest #{inspect(manifest, pretty: true, limit: :infinity, printable_limit: :infinity)} + + @documents #{inspect(documents, pretty: true, limit: :infinity, printable_limit: :infinity)} + + @impl true + def load(_opts), do: {:ok, @documents, @manifest} + + def documents, do: @documents + def manifest, do: @manifest + end + """ + end + + defp generated_at do + DateTime.utc_now() + |> DateTime.truncate(:second) + |> DateTime.to_iso8601() + end +end diff --git a/lib/tempest/application.ex b/lib/tempest/application.ex index a82e99b..c44c1b3 100644 --- a/lib/tempest/application.ex +++ b/lib/tempest/application.ex @@ -9,6 +9,7 @@ defmodule Tempest.Application do def start(_type, _args) do config = Tempest.Config.load!() Tempest.Storage.bootstrap!(config) + Tempest.Lexicon.Registry.validate_startup!() children = [ TempestWeb.Telemetry, diff --git a/lib/tempest/lexicon/bundled.ex b/lib/tempest/lexicon/bundled.ex new file mode 100644 index 0000000..db6bf32 --- /dev/null +++ b/lib/tempest/lexicon/bundled.ex @@ -0,0 +1,115 @@ +defmodule Tempest.Lexicon.Bundled do + @moduledoc """ + Bundled generated Lexicon documents. + + Regenerate with: + + mix tempest.lexicon.generate --source priv/lexicons/smoke --commit smoke-fixture + """ + + @behaviour Tempest.Lexicon.Provider + + @manifest %{ + "source_repo" => "atproto", + "source_commit" => "smoke-fixture", + "generated_at" => "2026-05-16T00:00:00Z", + "document_count" => 4, + "document_ids" => [ + "app.bsky.actor.profile", + "com.atproto.label.defs", + "com.atproto.lexicon.schema", + "com.atproto.repo.strongRef" + ] + } + + @documents [ + %{ + "defs" => %{ + "main" => %{ + "key" => "literal:self", + "record" => %{ + "properties" => %{ + "avatar" => %{"accept" => ["image/png", "image/jpeg"], "maxSize" => 1_000_000, "type" => "blob"}, + "banner" => %{"accept" => ["image/png", "image/jpeg"], "maxSize" => 1_000_000, "type" => "blob"}, + "createdAt" => %{"format" => "datetime", "type" => "string"}, + "description" => %{"maxGraphemes" => 256, "maxLength" => 2_560, "type" => "string"}, + "displayName" => %{"maxGraphemes" => 64, "maxLength" => 640, "type" => "string"}, + "joinedViaStarterPack" => %{"ref" => "com.atproto.repo.strongRef", "type" => "ref"}, + "labels" => %{"refs" => ["com.atproto.label.defs#selfLabels"], "type" => "union"}, + "pinnedPost" => %{"ref" => "com.atproto.repo.strongRef", "type" => "ref"}, + "pronouns" => %{"maxGraphemes" => 20, "maxLength" => 200, "type" => "string"}, + "website" => %{"format" => "uri", "type" => "string"} + }, + "type" => "object" + }, + "type" => "record" + } + }, + "id" => "app.bsky.actor.profile", + "lexicon" => 1 + }, + %{ + "defs" => %{ + "main" => %{ + "description" => "Representation of Lexicon schemas themselves, when published as atproto records.", + "key" => "nsid", + "record" => %{ + "properties" => %{ + "lexicon" => %{ + "description" => "Indicates the 'version' of the Lexicon language.", + "type" => "integer" + } + }, + "required" => ["lexicon"], + "type" => "object" + }, + "type" => "record" + } + }, + "id" => "com.atproto.lexicon.schema", + "lexicon" => 1 + }, + %{ + "defs" => %{ + "selfLabel" => %{ + "properties" => %{"val" => %{"maxLength" => 128, "type" => "string"}}, + "required" => ["val"], + "type" => "object" + }, + "selfLabels" => %{ + "properties" => %{ + "values" => %{ + "items" => %{"ref" => "#selfLabel", "type" => "ref"}, + "maxLength" => 10, + "type" => "array" + } + }, + "required" => ["values"], + "type" => "object" + } + }, + "id" => "com.atproto.label.defs", + "lexicon" => 1 + }, + %{ + "defs" => %{ + "main" => %{ + "properties" => %{ + "cid" => %{"format" => "cid", "type" => "string"}, + "uri" => %{"format" => "at-uri", "type" => "string"} + }, + "required" => ["uri", "cid"], + "type" => "object" + } + }, + "id" => "com.atproto.repo.strongRef", + "lexicon" => 1 + } + ] + + @impl true + def load(_opts), do: {:ok, @documents, @manifest} + + def documents, do: @documents + def manifest, do: @manifest +end diff --git a/lib/tempest/lexicon/document.ex b/lib/tempest/lexicon/document.ex new file mode 100644 index 0000000..f677ae3 --- /dev/null +++ b/lib/tempest/lexicon/document.ex @@ -0,0 +1,482 @@ +defmodule Tempest.Lexicon.Document do + @moduledoc """ + Generic validation and indexing for Lexicon schema documents. + + This module validates the shape and reference graph of Lexicon data. It does + not contain application-specific schemas. + """ + + alias Tempest.RepoCore.Nsid + + @default_limits [ + max_document_count: 1_000, + max_definitions_per_document: 1_000, + max_schema_depth: 64, + max_ref_depth: 64 + ] + + @schema_types MapSet.new( + ~w(array blob boolean bytes cid-link integer object params permission procedure query record ref string subscription token union unknown) + ) + + def validate_documents(documents, opts \\ []) + + def validate_documents(documents, opts) when is_list(documents) do + limits = Keyword.merge(@default_limits, opts) + + with :ok <- validate_document_count(documents, limits), + :ok <- validate_each_document(documents, limits), + :ok <- validate_unique_document_ids(documents), + :ok <- validate_unique_definition_refs(documents), + :ok <- validate_resolved_refs(documents), + :ok <- validate_ref_graph(documents, limits) do + :ok + end + end + + def validate_documents(_documents, _opts), do: {:error, :invalid_lexicon_documents} + + def validate_document(document, opts \\ []) + + def validate_document(%{"lexicon" => 1, "id" => id, "defs" => defs} = document, opts) + when is_binary(id) and is_map(defs) do + limits = Keyword.merge(@default_limits, opts) + + with :ok <- validate_nsid(id), + :ok <- validate_defs_count(id, defs, limits), + :ok <- validate_definitions(document, defs, limits) do + :ok + end + end + + def validate_document(%{"lexicon" => version}, _opts) when version != 1, + do: {:error, {:unsupported_lexicon_version, version}} + + def validate_document(_document, _opts), do: {:error, :invalid_lexicon_document} + + def definition_refs(%{"id" => id, "defs" => defs}) when is_binary(id) and is_map(defs) do + defs + |> Map.keys() + |> Enum.sort() + |> Enum.map(&build_ref(id, &1)) + end + + def definition_refs(_document), do: [] + + def referenced_definition_refs(%{"id" => id, "defs" => defs}) when is_binary(id) and is_map(defs) do + defs + |> Enum.flat_map(fn {_name, schema} -> collect_refs(schema, id) end) + |> Enum.uniq() + |> Enum.sort() + end + + def referenced_definition_refs(_document), do: [] + + def normalize_ref("#" <> name, current_id), do: build_ref(current_id, name) + def normalize_ref(ref, _current_id) when is_binary(ref), do: normalize_ref_string(ref) + + defp validate_document_count(documents, limits) do + if length(documents) <= limits[:max_document_count] do + :ok + else + {:error, {:loader_limit_exceeded, :max_document_count}} + end + end + + defp validate_each_document(documents, limits) do + Enum.reduce_while(documents, :ok, fn document, :ok -> + case validate_document(document, limits) do + :ok -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + defp validate_unique_document_ids(documents) do + documents + |> Enum.map(&Map.get(&1, "id")) + |> duplicates() + |> case do + [] -> :ok + ids -> {:error, {:duplicate_document_ids, ids}} + end + end + + defp validate_unique_definition_refs(documents) do + documents + |> Enum.flat_map(&definition_refs/1) + |> duplicates() + |> case do + [] -> :ok + refs -> {:error, {:duplicate_definition_refs, refs}} + end + end + + defp validate_resolved_refs(documents) do + known_refs = documents |> Enum.flat_map(&definition_refs/1) |> MapSet.new() + + documents + |> Enum.flat_map(&referenced_definition_refs/1) + |> Enum.reject(&MapSet.member?(known_refs, &1)) + |> Enum.uniq() + |> Enum.sort() + |> case do + [] -> :ok + refs -> {:error, {:unresolved_definition_refs, refs}} + end + end + + defp validate_ref_graph(documents, limits) do + graph = definition_ref_graph(documents) + + graph + |> Map.keys() + |> Enum.sort() + |> Enum.reduce_while(:ok, fn ref, :ok -> + case walk_ref_graph(ref, graph, [], limits[:max_ref_depth]) do + :ok -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + defp definition_ref_graph(documents) do + Map.new(documents, fn %{"id" => id, "defs" => defs} -> + defs = + Map.new(defs, fn {name, schema} -> + {build_ref(id, name), collect_refs(schema, id)} + end) + + {id, defs} + end) + |> Map.values() + |> Enum.reduce(%{}, &Map.merge/2) + end + + defp walk_ref_graph(ref, _graph, path, max_ref_depth) when length(path) > max_ref_depth do + {:error, {:loader_limit_exceeded, :max_ref_depth, Enum.reverse([ref | path])}} + end + + defp walk_ref_graph(ref, graph, path, max_ref_depth) do + cond do + ref in path -> + cycle = + path + |> Enum.reverse() + |> close_cycle(ref) + + {:error, {:ref_cycle, cycle}} + + true -> + graph + |> Map.get(ref, []) + |> Enum.reduce_while(:ok, fn next_ref, :ok -> + case walk_ref_graph(next_ref, graph, [ref | path], max_ref_depth) do + :ok -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + end + + defp close_cycle(path, ref) do + path + |> Enum.drop_while(&(&1 != ref)) + |> then(&(&1 ++ [ref])) + end + + defp validate_nsid(id) do + case Nsid.parse(id) do + {:ok, %Nsid{value: ^id}} -> :ok + {:error, _reason} -> {:error, {:invalid_lexicon_id, id}} + end + end + + defp validate_defs_count(id, defs, limits) do + cond do + map_size(defs) == 0 -> + {:error, {:missing_definitions, id}} + + map_size(defs) > limits[:max_definitions_per_document] -> + {:error, {:loader_limit_exceeded, :max_definitions_per_document}} + + true -> + :ok + end + end + + defp validate_definitions(%{"id" => id}, defs, limits) do + Enum.reduce_while(defs, :ok, fn {name, schema}, :ok -> + with :ok <- validate_definition_name(id, name), + :ok <- validate_schema(schema, id, "#{id}##{name}", 0, limits) do + {:cont, :ok} + else + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + defp validate_definition_name(_id, name) when is_binary(name) and name != "", do: :ok + defp validate_definition_name(id, name), do: {:error, {:invalid_definition_name, id, name}} + + defp validate_schema(schema, id, path, depth, limits) do + if depth > limits[:max_schema_depth] do + {:error, {:loader_limit_exceeded, :max_schema_depth, path}} + else + validate_schema_type(schema, id, path, depth, limits) + end + end + + defp validate_schema_type(%{"type" => type} = schema, id, path, depth, limits) when is_binary(type) do + cond do + not MapSet.member?(@schema_types, type) -> + {:error, {:unsupported_schema_type, path, type}} + + type == "record" -> + validate_record_schema(schema, id, path, depth, limits) + + type in ["query", "procedure"] -> + validate_xrpc_schema(schema, id, path, depth, limits) + + type == "subscription" -> + validate_subscription_schema(schema, id, path, depth, limits) + + type == "object" -> + validate_object_schema(schema, id, path, depth, limits) + + type == "params" -> + validate_object_schema(schema, id, path, depth, limits) + + type == "array" -> + validate_array_schema(schema, id, path, depth, limits) + + type == "ref" -> + validate_ref_schema(schema, id, path) + + type == "union" -> + validate_union_schema(schema, id, path) + + true -> + validate_primitive_schema(schema, path) + end + end + + defp validate_schema_type(_schema, _id, path, _depth, _limits), do: {:error, {:invalid_schema, path}} + + defp validate_record_schema(%{"record" => record} = schema, id, path, depth, limits) when is_map(record) do + with :ok <- validate_optional_key(schema, path) do + validate_schema(record, id, path <> ".record", depth + 1, limits) + end + end + + defp validate_record_schema(_schema, _id, path, _depth, _limits), do: {:error, {:invalid_schema, path}} + + defp validate_xrpc_schema(schema, id, path, depth, limits) do + with :ok <- validate_optional_parameters_schema(schema, id, path, depth, limits), + :ok <- validate_optional_io_schema(schema, "input", id, path, depth, limits), + :ok <- validate_optional_io_schema(schema, "output", id, path, depth, limits) do + :ok + end + end + + defp validate_subscription_schema(schema, id, path, depth, limits) do + validate_optional_io_schema(schema, "message", id, path, depth, limits) + end + + defp validate_object_schema(schema, id, path, depth, limits) do + with :ok <- validate_string_list(Map.get(schema, "required", []), path <> ".required"), + :ok <- validate_string_list(Map.get(schema, "nullable", []), path <> ".nullable"), + {:ok, properties} <- validate_optional_map(schema, "properties", path), + :ok <- validate_property_schemas(properties, id, path, depth, limits) do + :ok + end + end + + defp validate_array_schema(%{"items" => items}, id, path, depth, limits) when is_map(items), + do: validate_schema(items, id, path <> ".items", depth + 1, limits) + + defp validate_array_schema(_schema, _id, path, _depth, _limits), do: {:error, {:invalid_schema, path}} + + defp validate_optional_parameters_schema(%{"parameters" => parameters}, id, path, depth, limits) + when is_map(parameters), + do: validate_schema(parameters, id, path <> ".parameters", depth + 1, limits) + + defp validate_optional_parameters_schema(%{"parameters" => _parameters}, _id, path, _depth, _limits), + do: {:error, {:invalid_schema, path <> ".parameters"}} + + defp validate_optional_parameters_schema(_schema, _id, _path, _depth, _limits), do: :ok + + defp validate_optional_io_schema(schema, key, id, path, depth, limits) do + case Map.get(schema, key) do + nil -> + :ok + + %{"schema" => nested_schema} when is_map(nested_schema) -> + validate_schema(nested_schema, id, path <> "." <> key <> ".schema", depth + 1, limits) + + %{} -> + :ok + + _value -> + {:error, {:invalid_schema, path <> "." <> key}} + end + end + + defp validate_ref_schema(%{"ref" => ref}, id, path) when is_binary(ref), + do: validate_ref(ref, id, path) + + defp validate_ref_schema(_schema, _id, path), do: {:error, {:invalid_schema, path}} + + defp validate_union_schema(%{"refs" => refs}, id, path) when is_list(refs) and refs != [] do + with :ok <- validate_ref_list(refs, id, path <> ".refs") do + refs + |> Enum.map(&normalize_ref(&1, id)) + |> duplicates() + |> case do + [] -> :ok + duplicate_refs -> {:error, {:duplicate_refs, path, duplicate_refs}} + end + end + end + + defp validate_union_schema(_schema, _id, path), do: {:error, {:invalid_schema, path}} + + defp validate_primitive_schema(schema, path) do + with :ok <- validate_optional_string_list(schema, "knownValues", path), + :ok <- validate_optional_string_list(schema, "enum", path), + :ok <- validate_optional_integer(schema, "minimum", path), + :ok <- validate_optional_integer(schema, "maximum", path), + :ok <- validate_optional_integer(schema, "minLength", path), + :ok <- validate_optional_integer(schema, "maxLength", path), + :ok <- validate_optional_integer(schema, "minGraphemes", path), + :ok <- validate_optional_integer(schema, "maxGraphemes", path) do + :ok + end + end + + defp validate_optional_key(%{"key" => "any"}, _path), do: :ok + defp validate_optional_key(%{"key" => "tid"}, _path), do: :ok + defp validate_optional_key(%{"key" => "nsid"}, _path), do: :ok + defp validate_optional_key(%{"key" => "literal:" <> literal}, _path) when literal != "", do: :ok + defp validate_optional_key(%{"key" => key}, path), do: {:error, {:unsupported_record_key_type, path, key}} + defp validate_optional_key(_schema, _path), do: :ok + + defp validate_optional_map(schema, key, path) do + case Map.get(schema, key, %{}) do + value when is_map(value) -> {:ok, value} + _value -> {:error, {:invalid_schema, path <> "." <> key}} + end + end + + defp validate_property_schemas(properties, id, path, depth, limits) do + Enum.reduce_while(properties, :ok, fn {name, property_schema}, :ok -> + if is_binary(name) and name != "" do + case validate_schema(property_schema, id, path <> ".properties." <> name, depth + 1, limits) do + :ok -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + else + {:halt, {:error, {:invalid_property_name, path, name}}} + end + end) + end + + defp validate_ref_list(refs, id, path) do + Enum.reduce_while(refs, :ok, fn ref, :ok -> + case validate_ref(ref, id, path) do + :ok -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + + defp validate_ref(ref, id, path) when is_binary(ref) do + case normalize_ref(ref, id) do + nil -> {:error, {:invalid_ref, path, ref}} + _normalized_ref -> :ok + end + end + + defp validate_ref(ref, _id, path), do: {:error, {:invalid_ref, path, ref}} + + defp validate_optional_string_list(schema, key, path) do + case Map.get(schema, key) do + nil -> :ok + value -> validate_string_list(value, path <> "." <> key) + end + end + + defp validate_string_list(value, _path) when is_list(value) do + if Enum.all?(value, &is_binary/1), do: :ok, else: {:error, :invalid_lexicon_document} + end + + defp validate_string_list(_value, path), do: {:error, {:invalid_schema, path}} + + defp validate_optional_integer(schema, key, path) do + case Map.get(schema, key) do + nil -> :ok + value when is_integer(value) -> :ok + _value -> {:error, {:invalid_schema, path <> "." <> key}} + end + end + + defp collect_refs(%{"type" => "ref", "ref" => ref}, id) when is_binary(ref), do: [normalize_ref(ref, id)] + + defp collect_refs(%{"type" => "union", "refs" => refs}, id) when is_list(refs) do + refs + |> Enum.filter(&is_binary/1) + |> Enum.map(&normalize_ref(&1, id)) + end + + defp collect_refs(%{"type" => "record", "record" => record}, id) when is_map(record), do: collect_refs(record, id) + + defp collect_refs(%{"type" => type} = schema, id) when type in ["query", "procedure"] do + ["parameters", "input", "output"] + |> Enum.flat_map(fn key -> collect_io_refs(Map.get(schema, key), id) end) + end + + defp collect_refs(%{"type" => "subscription"} = schema, id), do: collect_io_refs(Map.get(schema, "message"), id) + + defp collect_refs(%{"type" => "object", "properties" => properties}, id) when is_map(properties) do + Enum.flat_map(properties, fn {_name, schema} -> collect_refs(schema, id) end) + end + + defp collect_refs(%{"type" => "array", "items" => items}, id) when is_map(items), do: collect_refs(items, id) + + defp collect_refs(_schema, _id), do: [] + + defp collect_io_refs(%{"schema" => schema}, id) when is_map(schema), do: collect_refs(schema, id) + defp collect_io_refs(%{"type" => _type} = schema, id), do: collect_refs(schema, id) + defp collect_io_refs(_schema, _id), do: [] + + defp normalize_ref_string("#" <> name) when name != "", do: "#" <> name + + defp normalize_ref_string(ref) do + case String.split(ref, "#", parts: 2) do + [id] -> + if valid_nsid_string?(id), do: id <> "#main" + + [id, name] -> + if valid_nsid_string?(id) and name != "", do: id <> "#" <> name + end + end + + defp build_ref(id, name) do + if valid_nsid_string?(id) and is_binary(name) and name != "" do + id <> "#" <> name + end + end + + defp valid_nsid_string?(id) do + match?({:ok, %Nsid{value: ^id}}, Nsid.parse(id)) + end + + defp duplicates(values) do + values + |> Enum.reject(&is_nil/1) + |> Enum.frequencies() + |> Enum.filter(fn {_value, count} -> count > 1 end) + |> Enum.map(fn {value, _count} -> value end) + |> Enum.sort() + end +end diff --git a/lib/tempest/lexicon/external_resolver.ex b/lib/tempest/lexicon/external_resolver.ex new file mode 100644 index 0000000..645ddfc --- /dev/null +++ b/lib/tempest/lexicon/external_resolver.ex @@ -0,0 +1,26 @@ +defmodule Tempest.Lexicon.ExternalResolver do + @moduledoc """ + Behaviour for policy-controlled external Lexicon resolution. + + External resolution is disabled by default. When enabled, the registry only + asks the configured resolver after bundled, generated, and operator-local + sources miss. A resolver must return a document whose `id` matches the + requested NSID, and the registry validates that document before trusting it. + + Source precedence is: + + 1. bundled generated schemas + 2. in-memory configured schemas, used by tests and controlled embedding + 3. operator-configured local schema files/directories + 4. external resolution, only when explicitly enabled + + External documents cannot override trusted local sources through this + interface because the registry consults the resolver only after local lookup + fails. Network resolution, caching, SSRF protections, and authority checks + belong in concrete resolver implementations. + """ + + @type resolve_result :: {:ok, map()} | {:error, :not_found | :disabled | term()} + + @callback resolve(String.t(), Keyword.t()) :: resolve_result() +end diff --git a/lib/tempest/lexicon/external_resolver/disabled.ex b/lib/tempest/lexicon/external_resolver/disabled.ex new file mode 100644 index 0000000..036873a --- /dev/null +++ b/lib/tempest/lexicon/external_resolver/disabled.ex @@ -0,0 +1,14 @@ +defmodule Tempest.Lexicon.ExternalResolver.Disabled do + @moduledoc """ + Default external Lexicon resolver. + + This resolver performs no network or dynamic lookup. It exists so the + registry has an explicit disabled policy path instead of treating missing + configuration as implicit permission to resolve schemas externally. + """ + + @behaviour Tempest.Lexicon.ExternalResolver + + @impl true + def resolve(_id, _opts), do: {:error, :disabled} +end diff --git a/lib/tempest/lexicon/external_resolver/network.ex b/lib/tempest/lexicon/external_resolver/network.ex new file mode 100644 index 0000000..66a01f2 --- /dev/null +++ b/lib/tempest/lexicon/external_resolver/network.ex @@ -0,0 +1,343 @@ +defmodule Tempest.Lexicon.ExternalResolver.Network do + @moduledoc """ + Network-backed resolver for published Lexicon schema records. + + The resolver follows the AT Protocol publication model: + + 1. derive the NSID authority by removing the final segment from the requested + schema id and reversing it into a domain; + 2. read `_lexicon.` DNS TXT records for a `did=` authority; + 3. resolve the DID document and find the `AtprotoPersonalDataServer` service; + 4. fetch the `com.atproto.lexicon.schema` record with rkey equal to the + requested NSID from that PDS. + + All outbound URLs pass through `Tempest.Identity.SsrfProtection`, redirects are + rejected, HTTP calls have conservative timeouts, responses are size-bounded, + and positive/negative cache entries are stored in ETS. Positive entries can be + served stale while a refresh fails; negative entries prevent repeated misses. + Concurrent refreshes for the same NSID are serialized with `:global.trans/2`. + """ + + @behaviour Tempest.Lexicon.ExternalResolver + + alias Tempest.Identity.{SsrfProtection, Validators} + + @cache_table :tempest_lexicon_external_resolver_cache + @txt_prefix "did=" + @schema_collection "com.atproto.lexicon.schema" + + @default_opts [ + positive_ttl_ms: 300_000, + negative_ttl_ms: 60_000, + stale_ttl_ms: 900_000, + max_response_bytes: 256_000, + receive_timeout: 2_000, + connect_timeout: 1_000, + req_options: [] + ] + + @impl true + def resolve(id, opts) when is_binary(id) do + opts = Keyword.merge(@default_opts, opts) + ensure_cache!() + now = monotonic_ms() + + case lookup_cache(id, now) do + {:fresh, document} -> {:ok, document} + {:negative, reason} -> {:error, reason} + {:stale, document} -> refresh_with_single_flight(id, opts, document) + :miss -> refresh_with_single_flight(id, opts, nil) + end + end + + def resolve(_id, _opts), do: {:error, :invalid_ref} + + def reset_cache! do + ensure_cache!() + :ets.delete_all_objects(@cache_table) + :ok + end + + defp refresh_with_single_flight(id, opts, stale_document) do + lock = {:lock, id} + acquire_lock(lock) + + try do + now = monotonic_ms() + + case lookup_cache(id, now) do + {:fresh, document} -> + {:ok, document} + + {:negative, reason} -> + {:error, reason} + + _miss_or_stale -> + case resolve_uncached(id, opts) do + {:ok, document} -> + put_positive_cache(id, document, opts) + {:ok, document} + + {:error, reason} -> + put_negative_cache(id, reason, opts) + + if is_map(stale_document) do + {:ok, stale_document} + else + {:error, reason} + end + end + end + after + :ets.delete(@cache_table, lock) + end + end + + defp acquire_lock(lock) do + if :ets.insert_new(@cache_table, {lock, self()}) do + :ok + else + Process.sleep(5) + acquire_lock(lock) + end + end + + defp resolve_uncached(id, opts) do + with {:ok, domain} <- authority_domain(id), + {:ok, did} <- resolve_lexicon_did(domain, opts), + {:ok, did_document} <- resolve_did_document(did, opts), + {:ok, service_endpoint} <- pds_service_endpoint(did_document), + :ok <- SsrfProtection.validate_url(service_endpoint), + {:ok, document} <- fetch_schema_record(service_endpoint, did, id, opts), + {:ok, ^id} <- fetch_matching_id(document, id) do + {:ok, document} + else + {:ok, other_id} -> {:error, {:resolved_lexicon_id_mismatch, id, other_id}} + {:error, reason} -> {:error, reason} + end + end + + defp authority_domain(id) do + parts = String.split(id, ".") + + if length(parts) >= 3 do + parts + |> Enum.drop(-1) + |> Enum.reverse() + |> Enum.join(".") + |> then(&{:ok, &1}) + else + {:error, :invalid_ref} + end + end + + defp resolve_lexicon_did(domain, opts) do + query = "_lexicon." <> domain + + query + |> dns_txt_lookup(opts) + |> Enum.find_value(fn record -> + record + |> txt_record_to_string() + |> parse_txt_did() + end) + |> case do + nil -> {:error, :not_found} + did -> validate_did(did) + end + end + + defp resolve_did_document(did, opts) do + case Keyword.get(opts, :did_document_lookup) do + fun when is_function(fun, 1) -> + fun.(did) + + nil -> + fetch_did_document(did, opts) + end + end + + defp fetch_did_document("did:web:" <> identifier, opts) do + host = String.replace(identifier, ":", ".") + url = "https://#{host}/.well-known/did.json" + + with :ok <- SsrfProtection.validate_url(url) do + fetch_json(url, opts) + end + end + + defp fetch_did_document("did:plc:" <> _identifier = did, opts) do + url = "https://plc.directory/#{URI.encode(did, &URI.char_unreserved?/1)}" + + with :ok <- SsrfProtection.validate_url(url) do + fetch_json(url, opts) + end + end + + defp fetch_did_document(_did, _opts), do: {:error, :unsupported_did_method} + + defp pds_service_endpoint(%{"service" => services}) when is_list(services) do + services + |> Enum.find_value(fn + %{"type" => "AtprotoPersonalDataServer", "serviceEndpoint" => endpoint} when is_binary(endpoint) -> endpoint + _service -> nil + end) + |> case do + nil -> {:error, :pds_service_not_found} + endpoint -> {:ok, endpoint} + end + end + + defp pds_service_endpoint(_document), do: {:error, :pds_service_not_found} + + defp fetch_schema_record(service_endpoint, did, id, opts) do + case Keyword.get(opts, :schema_record_lookup) do + fun when is_function(fun, 3) -> + with {:ok, %{"value" => value}} when is_map(value) <- fun.(service_endpoint, did, id) do + {:ok, Map.delete(value, "$type")} + else + {:ok, _body} -> {:error, :invalid_lexicon} + {:error, reason} -> {:error, reason} + end + + nil -> + query = + URI.encode_query(%{ + "repo" => did, + "collection" => @schema_collection, + "rkey" => id + }) + + url = String.trim_trailing(service_endpoint, "/") <> "/xrpc/com.atproto.repo.getRecord?" <> query + + with :ok <- SsrfProtection.validate_url(url), + {:ok, %{"value" => value}} when is_map(value) <- fetch_json(url, opts) do + {:ok, Map.delete(value, "$type")} + else + {:ok, _body} -> {:error, :invalid_lexicon} + {:error, reason} -> {:error, reason} + end + end + end + + defp fetch_matching_id(%{"id" => id}, id), do: {:ok, id} + defp fetch_matching_id(%{"id" => other_id}, _id) when is_binary(other_id), do: {:ok, other_id} + defp fetch_matching_id(_document, _id), do: {:error, :invalid_lexicon} + + defp fetch_json(url, opts) do + req_opts = + [ + url: url, + redirect: false, + retry: false, + receive_timeout: opts[:receive_timeout], + connect_options: [timeout: opts[:connect_timeout]] + ] + |> Keyword.merge(opts[:req_options]) + + case Req.get(req_opts) do + {:ok, %{status: 200, body: body}} -> + with :ok <- validate_response_size(body, opts), + {:ok, decoded} <- decode_json_body(body) do + {:ok, decoded} + end + + {:ok, %{status: status}} when status in [301, 302, 303, 307, 308] -> + {:error, :redirect_rejected} + + {:ok, %{status: 404}} -> + {:error, :not_found} + + {:ok, _response} -> + {:error, :resolution_failed} + + {:error, _reason} -> + {:error, :resolution_failed} + end + end + + defp validate_response_size(body, opts) when is_binary(body) do + if byte_size(body) <= opts[:max_response_bytes], do: :ok, else: {:error, :response_too_large} + end + + defp validate_response_size(body, opts) do + body + |> Jason.encode!() + |> validate_response_size(opts) + end + + defp decode_json_body(body) when is_binary(body) do + case Jason.decode(body) do + {:ok, decoded} -> {:ok, decoded} + {:error, _reason} -> {:error, :invalid_json} + end + end + + defp decode_json_body(body) when is_map(body), do: {:ok, body} + defp decode_json_body(_body), do: {:error, :invalid_json} + + defp lookup_cache(id, now) do + case :ets.lookup(@cache_table, id) do + [{^id, {:positive, document, expires_at, _stale_until}}] when now <= expires_at -> + {:fresh, document} + + [{^id, {:positive, document, _expires_at, stale_until}}] when now <= stale_until -> + {:stale, document} + + [{^id, {:negative, reason, expires_at}}] when now <= expires_at -> + {:negative, reason} + + _other -> + :miss + end + end + + defp put_positive_cache(id, document, opts) do + now = monotonic_ms() + expires_at = now + opts[:positive_ttl_ms] + stale_until = expires_at + opts[:stale_ttl_ms] + :ets.insert(@cache_table, {id, {:positive, document, expires_at, stale_until}}) + end + + defp put_negative_cache(id, reason, opts) do + :ets.insert(@cache_table, {id, {:negative, reason, monotonic_ms() + opts[:negative_ttl_ms]}}) + end + + defp ensure_cache! do + case :ets.whereis(@cache_table) do + :undefined -> :ets.new(@cache_table, [:named_table, :public, read_concurrency: true]) + _tid -> @cache_table + end + end + + defp monotonic_ms, do: System.monotonic_time(:millisecond) + + defp dns_txt_lookup(query, opts) do + case Keyword.get(opts, :dns_txt_lookup) do + nil -> :inet_res.lookup(String.to_charlist(query), :in, :txt) + fun when is_function(fun, 1) -> fun.(query) + {module, function, args} -> apply(module, function, [query | args]) + end + rescue + _error -> [] + end + + defp txt_record_to_string(record) when is_list(record) do + record + |> List.flatten() + |> List.to_string() + end + + defp txt_record_to_string(record) when is_binary(record), do: record + defp txt_record_to_string(_record), do: "" + + defp parse_txt_did(@txt_prefix <> did), do: String.trim(did) + defp parse_txt_did(_record), do: nil + + defp validate_did(did) do + case Validators.validate_did(did) do + :ok -> {:ok, did} + {:error, reason} -> {:error, reason} + end + end +end diff --git a/lib/tempest/lexicon/local_provider.ex b/lib/tempest/lexicon/local_provider.ex new file mode 100644 index 0000000..524c42a --- /dev/null +++ b/lib/tempest/lexicon/local_provider.ex @@ -0,0 +1,92 @@ +defmodule Tempest.Lexicon.LocalProvider do + @moduledoc """ + Loads operator-configured Lexicon JSON files from local files or directories. + """ + + @behaviour Tempest.Lexicon.Provider + + @default_limits [ + max_files: 1_000, + max_file_bytes: 1_000_000 + ] + + @impl true + def load(opts) do + paths = Keyword.get(opts, :paths, []) + limits = Keyword.merge(@default_limits, opts) + + with {:ok, files} <- expand_files(paths, limits), + {:ok, documents} <- read_documents(files, limits) do + {:ok, documents, local_manifest(files, documents)} + end + end + + defp expand_files(paths, limits) when is_list(paths) do + files = + paths + |> Enum.flat_map(&lexicon_files/1) + |> Enum.uniq() + |> Enum.sort() + + if length(files) <= limits[:max_files] do + {:ok, files} + else + {:error, {:loader_limit_exceeded, :max_files}} + end + end + + defp expand_files(_paths, _limits), do: {:error, :invalid_lexicon_paths} + + defp read_documents(files, limits) do + Enum.reduce_while(files, {:ok, []}, fn file, {:ok, documents} -> + case read_document(file, limits) do + {:ok, document} -> {:cont, {:ok, [document | documents]}} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + |> case do + {:ok, documents} -> {:ok, Enum.reverse(documents)} + {:error, reason} -> {:error, reason} + end + end + + defp read_document(file, limits) do + with {:ok, stat} <- File.stat(file), + :ok <- validate_file_size(file, stat.size, limits), + {:ok, json} <- File.read(file), + {:ok, document} <- Jason.decode(json) do + if is_map(document), do: {:ok, document}, else: {:error, {:invalid_lexicon_json, file}} + else + {:error, %Jason.DecodeError{} = reason} -> {:error, {:invalid_lexicon_json, file, reason.data}} + {:error, {:loader_limit_exceeded, _limit, _detail} = reason} -> {:error, reason} + {:error, reason} -> {:error, {:lexicon_file_error, file, reason}} + end + end + + defp validate_file_size(file, size, limits) do + if size <= limits[:max_file_bytes] do + :ok + else + {:error, {:loader_limit_exceeded, :max_file_bytes, file}} + end + end + + defp lexicon_files(path) when is_binary(path) do + cond do + File.dir?(path) -> Path.wildcard(Path.join(path, "**/*.json")) + File.regular?(path) -> [path] + true -> [] + end + end + + defp lexicon_files(_path), do: [] + + defp local_manifest(files, documents) do + %{ + "source" => "local", + "file_count" => length(files), + "document_count" => length(documents), + "document_ids" => documents |> Enum.map(&Map.get(&1, "id")) |> Enum.sort() + } + end +end diff --git a/lib/tempest/lexicon/provider.ex b/lib/tempest/lexicon/provider.ex new file mode 100644 index 0000000..2de400b --- /dev/null +++ b/lib/tempest/lexicon/provider.ex @@ -0,0 +1,14 @@ +defmodule Tempest.Lexicon.Provider do + @moduledoc """ + Boundary for trusted local Lexicon document sources. + + Providers return decoded Lexicon documents plus optional source metadata. + The registry owns validation, duplicate detection, and source composition + so record validation never needs to know where a schema came from. + """ + + @type manifest :: map() + @type load_result :: {:ok, [map()], manifest() | nil} | {:error, term()} + + @callback load(Keyword.t()) :: load_result() +end diff --git a/lib/tempest/lexicon/registry.ex b/lib/tempest/lexicon/registry.ex index b1a7503..a9685b0 100644 --- a/lib/tempest/lexicon/registry.ex +++ b/lib/tempest/lexicon/registry.ex @@ -1,14 +1,40 @@ defmodule Tempest.Lexicon.Registry do @moduledoc """ - Runtime lookup boundary for Lexicon documents. + Runtime lookup boundary for known Lexicon documents. + + The registry composes trusted local providers, validates their documents as + a set, and exposes deterministic lookup by document id or definition ref. + + Source precedence is bundled generated schemas, configured in-memory documents, + operator-configured local files/directories, then the configured external resolver + if `external_resolver: [enabled?: true]` is set. + + External resolution is disabled by default. Even when enabled, a resolver is + only consulted after local sources miss, so dynamic schemas cannot override + bundled or operator-local schemas through the normal lookup path. """ + alias Tempest.Lexicon.Document + @env_key __MODULE__ + @default_external_resolver [ + enabled?: false, + resolver: Tempest.Lexicon.ExternalResolver.Network, + opts: [] + ] + @default_config [ + bundled?: true, + bundled_provider: Tempest.Lexicon.Bundled, + documents: [], + paths: [], + limits: [], + external_resolver: @default_external_resolver + ] def fetch(id) when is_binary(id) do - case Map.fetch(documents(), id) do + case Map.fetch(local_documents!(), id) do {:ok, document} -> {:ok, document} - :error -> {:error, :unknown_lexicon} + :error -> fetch_external(id) end end @@ -33,57 +59,130 @@ defmodule Tempest.Lexicon.Registry do def normalize_ref("#" <> name, current_id) when is_binary(current_id), do: current_id <> "#" <> name def normalize_ref(ref, _current_id) when is_binary(ref), do: ref - defp documents do - config = Application.get_env(:tempest, @env_key, []) - - config - |> Keyword.get(:documents, %{}) - |> normalize_documents() - |> Map.merge(load_path_documents(Keyword.get(config, :paths, []))) - end - - defp load_path_documents(paths) when is_list(paths) do - paths - |> Enum.flat_map(&lexicon_files/1) - |> Enum.reduce(%{}, fn path, documents -> - case File.read(path) do - {:ok, json} -> - case Jason.decode(json) do - {:ok, %{"id" => id} = document} -> Map.put(documents, id, document) - {:ok, _value} -> documents - {:error, _reason} -> documents - end - - {:error, _reason} -> - documents - end - end) + def manifest do + config() + |> load_sources() + |> case do + {:ok, _documents, manifests} -> {:ok, manifests} + {:error, reason} -> {:error, reason} + end end - defp load_path_documents(_paths), do: %{} + def validate_startup! do + config() + |> load_sources() + |> case do + {:ok, documents, _manifests} -> + case Document.validate_documents(documents, limits()) do + :ok -> :ok + {:error, reason} -> raise ArgumentError, "invalid Lexicon documents: #{inspect(reason)}" + end + + {:error, reason} -> + raise ArgumentError, "invalid Lexicon registry configuration: #{inspect(reason)}" + end + end - defp lexicon_files(path) when is_binary(path) do - cond do - File.dir?(path) -> Path.wildcard(Path.join(path, "**/*.json")) - File.regular?(path) -> [path] - true -> [] + def validate_config(config) when is_list(config) do + @default_config + |> Keyword.merge(config) + |> load_sources() + |> case do + {:ok, documents, _manifests} -> Document.validate_documents(documents, Keyword.get(config, :limits, [])) + {:error, reason} -> {:error, reason} end end - defp lexicon_files(_path), do: [] + defp local_documents! do + config() + |> load_sources() + |> case do + {:ok, documents, _manifests} -> + case Document.validate_documents(documents, limits()) do + :ok -> Map.new(documents, &{Map.fetch!(&1, "id"), &1}) + {:error, reason} -> raise ArgumentError, "invalid Lexicon documents: #{inspect(reason)}" + end + + {:error, reason} -> + raise ArgumentError, "invalid Lexicon registry configuration: #{inspect(reason)}" + end + end - defp normalize_documents(documents) when is_map(documents) do - Map.new(documents, fn - {id, %{"id" => id} = document} -> {id, document} - {_key, %{"id" => id} = document} -> {id, document} - end) + defp fetch_external(id) do + config = config() + resolver_config = Keyword.merge(@default_external_resolver, Keyword.get(config, :external_resolver, [])) + + if Keyword.get(resolver_config, :enabled?, false) do + resolver = Keyword.fetch!(resolver_config, :resolver) + opts = Keyword.get(resolver_config, :opts, []) + + with {:ok, %{"id" => ^id} = document} <- resolver.resolve(id, opts), + :ok <- Document.validate_documents([document], limits()) do + {:ok, document} + else + {:ok, %{"id" => _other_id}} -> {:error, :unknown_lexicon} + {:error, _reason} -> {:error, :unknown_lexicon} + _other -> {:error, :invalid_lexicon} + end + else + {:error, :unknown_lexicon} + end + end + + defp config do + Keyword.merge(@default_config, Application.get_env(:tempest, @env_key, [])) + end + + defp limits do + Keyword.get(config(), :limits, []) + end + + defp load_sources(config) do + with {:ok, bundled_documents, bundled_manifest} <- load_bundled(config), + {:ok, configured_documents} <- normalize_documents(Keyword.get(config, :documents, [])), + {:ok, local_documents, local_manifest} <- + Tempest.Lexicon.LocalProvider.load( + Keyword.merge(Keyword.get(config, :limits, []), paths: Keyword.get(config, :paths, [])) + ) do + documents = bundled_documents ++ configured_documents ++ local_documents + + manifests = + [bundled_manifest, local_manifest] + |> Enum.reject(&is_nil/1) + |> Enum.reject(&(Map.get(&1, "source") == "local" and Map.get(&1, "file_count") == 0)) + + {:ok, documents, manifests} + end + end + + defp load_bundled(config) do + if Keyword.get(config, :bundled?, true) do + provider = Keyword.fetch!(config, :bundled_provider) + provider.load(Keyword.get(config, :limits, [])) + else + {:ok, [], nil} + end end defp normalize_documents(documents) when is_list(documents) do - Map.new(documents, fn %{"id" => id} = document -> {id, document} end) + if Enum.all?(documents, &is_map/1) do + {:ok, documents} + else + {:error, :invalid_lexicon_documents} + end + end + + defp normalize_documents(documents) when is_map(documents) do + values = Map.values(documents) + + if Enum.all?(values, &is_map/1) do + {:ok, values} + else + {:error, :invalid_lexicon_documents} + end end - defp normalize_documents(_documents), do: %{} + defp normalize_documents(_documents), do: {:error, :invalid_lexicon_documents} defp parse_ref("#" <> name, current_id) when is_binary(current_id), do: {:ok, current_id, name} defp parse_ref("#" <> _name, nil), do: {:error, :relative_ref_without_context} diff --git a/lib/tempest/lexicon/validator.ex b/lib/tempest/lexicon/validator.ex index e0f0d6b..9db6562 100644 --- a/lib/tempest/lexicon/validator.ex +++ b/lib/tempest/lexicon/validator.ex @@ -15,25 +15,25 @@ defmodule Tempest.Lexicon.Validator do require_schema? = Keyword.get(opts, :require_schema?, false) with :ok <- validate_record_type(collection, record) do - case Registry.fetch_record(collection) do - {:ok, document, definition} -> - with :ok <- validate_record_key(Map.get(definition, "key"), rkey) do - if validate_schema? do + if validate_schema? do + case Registry.fetch_record(collection) do + {:ok, document, definition} -> + with :ok <- validate_record_key(Map.get(definition, "key"), rkey) do schema = Map.fetch!(definition, "record") with :ok <- validate_value(record, schema, document, collection, 0) do {:ok, :valid} end - else - {:ok, :unknown} end - end - {:error, :unknown_lexicon} when not require_schema? -> - {:ok, :unknown} + {:error, :unknown_lexicon} when not require_schema? -> + {:ok, :unknown} - {:error, reason} -> - {:error, reason} + {:error, reason} -> + {:error, reason} + end + else + {:ok, :unknown} end end end @@ -47,6 +47,10 @@ defmodule Tempest.Lexicon.Validator do defp validate_record_key("literal:" <> literal, literal), do: :ok defp validate_record_key("literal:" <> literal, _rkey), do: {:error, {:invalid_record_key, "literal:" <> literal}} defp validate_record_key("tid", rkey), do: if(Tid.valid?(rkey), do: :ok, else: {:error, {:invalid_record_key, "tid"}}) + + defp validate_record_key("nsid", rkey), + do: if(match?({:ok, _nsid}, Nsid.parse(rkey)), do: :ok, else: {:error, {:invalid_record_key, "nsid"}}) + defp validate_record_key("any", _rkey), do: :ok defp validate_record_key(nil, _rkey), do: :ok defp validate_record_key(key_type, _rkey), do: {:error, {:unsupported_record_key_type, key_type}} diff --git a/test/smoke/lexicon-schemas.hurl b/test/smoke/lexicon-schemas.hurl new file mode 100644 index 0000000..15a0979 --- /dev/null +++ b/test/smoke/lexicon-schemas.hurl @@ -0,0 +1,66 @@ +POST {{base_url}}/xrpc/com.atproto.server.createAccount +Content-Type: application/json +{ + "handle": "lexicon-{{newUuid}}.test", + "email": "lexicon-{{newUuid}}@example.com", + "password": "correct horse battery staple" +} +HTTP 200 +[Captures] +created_did: jsonpath "$.did" +access_token: jsonpath "$.accessJwt" +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.did" startsWith "did:plc:" + +POST {{base_url}}/xrpc/com.atproto.repo.createRecord +Authorization: Bearer {{access_token}} +Content-Type: application/json +{ + "repo": "{{created_did}}", + "collection": "com.atproto.lexicon.schema", + "rkey": "example.lexicon.smoke", + "record": { + "$type": "com.atproto.lexicon.schema", + "lexicon": 1 + } +} +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.validationStatus" == "valid" + +POST {{base_url}}/xrpc/com.atproto.repo.createRecord +Authorization: Bearer {{access_token}} +Content-Type: application/json +{ + "repo": "{{created_did}}", + "collection": "example.lexicon.smoke", + "rkey": "one", + "record": { + "$type": "example.lexicon.smoke", + "text": "optimistic unknown" + } +} +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.validationStatus" == "unknown" + +POST {{base_url}}/xrpc/com.atproto.repo.createRecord +Authorization: Bearer {{access_token}} +Content-Type: application/json +{ + "repo": "{{created_did}}", + "collection": "example.lexicon.strict", + "rkey": "one", + "validate": true, + "record": { + "$type": "example.lexicon.strict", + "text": "strict unknown" + } +} +HTTP 400 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.error" == "InvalidRequest" diff --git a/test/support/lexicon_fixtures.ex b/test/support/lexicon_fixtures.ex index 852f1b4..8a3d961 100644 --- a/test/support/lexicon_fixtures.ex +++ b/test/support/lexicon_fixtures.ex @@ -4,7 +4,7 @@ defmodule Tempest.LexiconFixtures do def install!(test_context) do previous_config = Application.get_env(:tempest, Tempest.Lexicon.Registry, []) - Application.put_env(:tempest, Tempest.Lexicon.Registry, documents: documents()) + Application.put_env(:tempest, Tempest.Lexicon.Registry, bundled?: false, documents: documents()) ExUnit.Callbacks.on_exit(test_context, fn -> Application.put_env(:tempest, Tempest.Lexicon.Registry, previous_config) diff --git a/test/tempest/lexicon/com_atproto_compat_test.exs b/test/tempest/lexicon/com_atproto_compat_test.exs new file mode 100644 index 0000000..327780a --- /dev/null +++ b/test/tempest/lexicon/com_atproto_compat_test.exs @@ -0,0 +1,168 @@ +defmodule Tempest.Lexicon.OfficialComAtprotoCompatibilityTest do + use ExUnit.Case, async: true + + alias Tempest.Lexicon.Document + + @moduledoc false + + @official_com_atproto_subset [ + %{ + "lexicon" => 1, + "id" => "com.atproto.repo.strongRef", + "description" => "A URI with a content-hash fingerprint.", + "defs" => %{ + "main" => %{ + "type" => "object", + "required" => ["uri", "cid"], + "properties" => %{ + "uri" => %{"type" => "string", "format" => "at-uri"}, + "cid" => %{"type" => "string", "format" => "cid"} + } + } + } + }, + %{ + "lexicon" => 1, + "id" => "com.atproto.repo.defs", + "defs" => %{ + "commitMeta" => %{ + "type" => "object", + "required" => ["cid", "rev"], + "properties" => %{ + "cid" => %{"type" => "string", "format" => "cid"}, + "rev" => %{"type" => "string", "format" => "tid"} + } + } + } + }, + %{ + "lexicon" => 1, + "id" => "com.atproto.lexicon.schema", + "defs" => %{ + "main" => %{ + "type" => "record", + "description" => "Representation of Lexicon schemas themselves, when published as atproto records.", + "key" => "nsid", + "record" => %{ + "type" => "object", + "required" => ["lexicon"], + "properties" => %{ + "lexicon" => %{ + "type" => "integer", + "description" => "Indicates the 'version' of the Lexicon language." + } + } + } + } + } + }, + %{ + "lexicon" => 1, + "id" => "com.atproto.repo.createRecord", + "defs" => %{ + "main" => %{ + "type" => "procedure", + "description" => "Create a single new repository record. Requires auth, implemented by PDS.", + "input" => %{ + "encoding" => "application/json", + "schema" => %{ + "type" => "object", + "required" => ["repo", "collection", "record"], + "properties" => %{ + "repo" => %{"type" => "string", "format" => "at-identifier"}, + "collection" => %{"type" => "string", "format" => "nsid"}, + "rkey" => %{"type" => "string", "format" => "record-key", "maxLength" => 512}, + "validate" => %{"type" => "boolean"}, + "record" => %{"type" => "unknown"}, + "swapCommit" => %{"type" => "string", "format" => "cid"} + } + } + }, + "output" => %{ + "encoding" => "application/json", + "schema" => %{ + "type" => "object", + "required" => ["uri", "cid"], + "properties" => %{ + "uri" => %{"type" => "string", "format" => "at-uri"}, + "cid" => %{"type" => "string", "format" => "cid"}, + "commit" => %{"type" => "ref", "ref" => "com.atproto.repo.defs#commitMeta"}, + "validationStatus" => %{"type" => "string", "knownValues" => ["valid", "unknown"]} + } + } + }, + "errors" => [%{"name" => "InvalidSwap"}] + } + } + }, + %{ + "lexicon" => 1, + "id" => "com.atproto.identity.resolveHandle", + "defs" => %{ + "main" => %{ + "type" => "query", + "description" => "Resolves an atproto handle (hostname) to a DID.", + "parameters" => %{ + "type" => "params", + "required" => ["handle"], + "properties" => %{"handle" => %{"type" => "string", "format" => "handle"}} + }, + "output" => %{ + "encoding" => "application/json", + "schema" => %{ + "type" => "object", + "required" => ["did"], + "properties" => %{"did" => %{"type" => "string", "format" => "did"}} + } + }, + "errors" => [%{"name" => "HandleNotFound"}] + } + } + }, + %{ + "lexicon" => 1, + "id" => "com.atproto.sync.getLatestCommit", + "defs" => %{ + "main" => %{ + "type" => "query", + "description" => "Get the current commit CID & revision of the specified repo.", + "parameters" => %{ + "type" => "params", + "required" => ["did"], + "properties" => %{"did" => %{"type" => "string", "format" => "did"}} + }, + "output" => %{ + "encoding" => "application/json", + "schema" => %{ + "type" => "object", + "required" => ["cid", "rev"], + "properties" => %{ + "cid" => %{"type" => "string", "format" => "cid"}, + "rev" => %{"type" => "string", "format" => "tid"} + } + } + }, + "errors" => [ + %{"name" => "RepoNotFound"}, + %{"name" => "RepoTakendown"}, + %{"name" => "RepoSuspended"}, + %{"name" => "RepoDeactivated"} + ] + } + } + } + ] + + test "validates official com.atproto Lexicons relevant to Tempest" do + assert :ok = Document.validate_documents(@official_com_atproto_subset) + end + + test "compatibility target excludes official app.bsky profile post and follow records" do + ids = Enum.map(@official_com_atproto_subset, &Map.fetch!(&1, "id")) + + assert Enum.all?(ids, &String.starts_with?(&1, "com.atproto.")) + refute "app.bsky.actor.profile" in ids + refute "app.bsky.feed.post" in ids + refute "app.bsky.graph.follow" in ids + end +end diff --git a/test/tempest/lexicon/external_resolver_network_test.exs b/test/tempest/lexicon/external_resolver_network_test.exs new file mode 100644 index 0000000..8ed2336 --- /dev/null +++ b/test/tempest/lexicon/external_resolver_network_test.exs @@ -0,0 +1,197 @@ +defmodule Tempest.Lexicon.ExternalResolver.NetworkTest do + use ExUnit.Case, async: false + + import Plug.Conn + + alias Tempest.Lexicon.ExternalResolver.Network + + @did "did:plc:abcdefghijklmnopqrstuvwxyz234567" + @id "example.remote.record" + @query "_lexicon.remote.example" + @document %{ + "lexicon" => 1, + "id" => @id, + "defs" => %{ + "main" => %{ + "type" => "record", + "key" => "any", + "record" => %{"type" => "object", "properties" => %{"text" => %{"type" => "string"}}} + } + } + } + + setup context do + Req.Test.set_req_test_from_context(context) + Req.Test.verify_on_exit!(context) + Network.reset_cache!() + + old_identity_config = Application.get_env(:tempest, Tempest.Identity, []) + + Application.put_env(:tempest, Tempest.Identity, + dns_lookup: fn + "pds.example" -> {:ok, [{93, 184, 216, 34}]} + "private.example" -> {:ok, [{127, 0, 0, 1}]} + "plc.directory" -> {:ok, [{93, 184, 216, 34}]} + _host -> {:error, :nxdomain} + end + ) + + on_exit(fn -> + Network.reset_cache!() + Application.put_env(:tempest, Tempest.Identity, old_identity_config) + end) + + :ok + end + + test "resolves NSID authority through DNS, DID document, and PDS schema record" do + Req.Test.expect(__MODULE__, fn conn -> + assert conn.scheme == :https + assert conn.host == "pds.example" + assert conn.request_path == "/xrpc/com.atproto.repo.getRecord" + assert conn.query_string =~ "collection=com.atproto.lexicon.schema" + assert conn.query_string =~ "rkey=example.remote.record" + + json(conn, %{"value" => Map.put(@document, "$type", "com.atproto.lexicon.schema")}) + end) + + assert {:ok, @document} = Network.resolve(@id, resolver_opts()) + end + + test "caches positive resolution results" do + counter = start_counter!() + + opts = + resolver_opts( + schema_record_lookup: fn _endpoint, _did, _id -> + increment_counter(counter) + {:ok, %{"value" => @document}} + end + ) + + assert {:ok, @document} = Network.resolve(@id, opts) + assert {:ok, @document} = Network.resolve(@id, opts) + assert counter_value(counter) == 1 + end + + test "caches negative resolution results" do + counter = start_counter!() + + opts = + resolver_opts( + dns_txt_lookup: fn @query -> + increment_counter(counter) + [] + end + ) + + assert {:error, :not_found} = Network.resolve(@id, opts) + assert {:error, :not_found} = Network.resolve(@id, opts) + assert counter_value(counter) == 1 + end + + test "serves stale positive cache when refresh fails" do + counter = start_counter!() + + opts = + resolver_opts( + positive_ttl_ms: -1, + stale_ttl_ms: 60_000, + schema_record_lookup: fn _endpoint, _did, _id -> + case increment_counter(counter) do + 1 -> {:ok, %{"value" => @document}} + _count -> {:error, :resolution_failed} + end + end + ) + + assert {:ok, @document} = Network.resolve(@id, opts) + assert {:ok, @document} = Network.resolve(@id, opts) + assert counter_value(counter) == 2 + end + + test "serializes concurrent refreshes for the same schema id" do + counter = start_counter!() + parent = self() + + opts = + resolver_opts( + schema_record_lookup: fn _endpoint, _did, _id -> + send(parent, :fetch_started) + increment_counter(counter) + Process.sleep(25) + {:ok, %{"value" => @document}} + end + ) + + tasks = for _index <- 1..2, do: Task.async(fn -> Network.resolve(@id, opts) end) + + assert {:ok, @document} = Task.await(Enum.at(tasks, 0)) + assert {:ok, @document} = Task.await(Enum.at(tasks, 1)) + assert_receive :fetch_started + refute_receive :fetch_started, 50 + assert counter_value(counter) == 1 + end + + test "rejects private PDS service endpoints before fetching schema records" do + opts = + resolver_opts( + did_document_lookup: fn @did -> + {:ok, + %{"service" => [%{"type" => "AtprotoPersonalDataServer", "serviceEndpoint" => "https://private.example"}]}} + end + ) + + assert {:error, :private_ip} = Network.resolve(@id, opts) + end + + test "rejects redirects and oversized responses" do + Req.Test.expect(__MODULE__, fn conn -> + conn + |> put_resp_header("location", "https://pds.example/elsewhere") + |> send_resp(302, "") + end) + + assert {:error, :redirect_rejected} = Network.resolve(@id, resolver_opts()) + + Network.reset_cache!() + + Req.Test.expect(__MODULE__, fn conn -> + send_resp(conn, 200, String.duplicate("x", 128)) + end) + + assert {:error, :response_too_large} = Network.resolve(@id, resolver_opts(max_response_bytes: 32)) + end + + defp resolver_opts(overrides \\ []) do + Keyword.merge( + [ + dns_txt_lookup: fn @query -> ["did=#{@did}"] end, + did_document_lookup: fn @did -> + {:ok, %{"service" => [%{"type" => "AtprotoPersonalDataServer", "serviceEndpoint" => "https://pds.example"}]}} + end, + req_options: [plug: {Req.Test, __MODULE__}] + ], + overrides + ) + end + + defp json(conn, body) do + conn + |> put_resp_header("content-type", "application/json") + |> send_resp(200, Jason.encode!(body)) + end + + defp start_counter! do + start_supervised!({Agent, fn -> 0 end}) + end + + defp increment_counter(counter) do + Agent.get_and_update(counter, fn value -> + next = value + 1 + {next, next} + end) + end + + defp counter_value(counter), do: Agent.get(counter, & &1) +end diff --git a/test/tempest/lexicon/generate_task_test.exs b/test/tempest/lexicon/generate_task_test.exs new file mode 100644 index 0000000..68bbf17 --- /dev/null +++ b/test/tempest/lexicon/generate_task_test.exs @@ -0,0 +1,69 @@ +defmodule Tempest.Lexicon.GenerateTaskTest do + use ExUnit.Case, async: false + + alias Mix.Tasks.Tempest.Lexicon.Generate + + test "generates deterministic bundled module with manifest and ref dependencies" do + source = tmp_dir!("source") + output = Path.join(tmp_dir!("output"), "bundled.ex") + + write_json!( + Path.join(source, "example.app.note.json"), + lexicon("example.app.note", %{ + "main" => %{ + "type" => "record", + "key" => "any", + "record" => %{ + "type" => "object", + "properties" => %{"subject" => %{"type" => "ref", "ref" => "example.app.subject"}} + } + } + }) + ) + + write_json!( + Path.join(source, "example.app.subject.json"), + lexicon("example.app.subject", %{ + "main" => %{ + "type" => "object", + "required" => ["name"], + "properties" => %{"name" => %{"type" => "string"}} + } + }) + ) + + Generate.run([ + "--source", + source, + "--commit", + "abc123", + "--source-repo", + "example", + "--generated-at", + "2026-05-16T12:00:00Z", + "--include", + "example.app.note", + "--out", + output + ]) + + generated = File.read!(output) + + assert generated =~ ~s("source_repo" => "example") + assert generated =~ ~s("source_commit" => "abc123") + assert generated =~ ~s("generated_at" => "2026-05-16T12:00:00Z") + assert generated =~ ~s("document_count" => 2) + assert generated =~ ~s("example.app.note") + assert generated =~ ~s("example.app.subject") + end + + defp lexicon(id, defs), do: %{"lexicon" => 1, "id" => id, "defs" => defs} + + defp tmp_dir!(name) do + path = Path.join(System.tmp_dir!(), "tempest-generate-#{name}-#{System.unique_integer([:positive])}") + File.mkdir_p!(path) + path + end + + defp write_json!(path, data), do: File.write!(path, Jason.encode!(data)) +end diff --git a/test/tempest/lexicon/registry_test.exs b/test/tempest/lexicon/registry_test.exs index cab5443..fc15249 100644 --- a/test/tempest/lexicon/registry_test.exs +++ b/test/tempest/lexicon/registry_test.exs @@ -1,21 +1,253 @@ defmodule Tempest.Lexicon.RegistryTest do use ExUnit.Case, async: false + alias Tempest.Lexicon.Document alias Tempest.Lexicon.Registry - test "loads Lexicon documents from configured fixture paths" do + setup do previous_config = Application.get_env(:tempest, Registry, []) - Application.put_env(:tempest, Registry, paths: [Path.expand("../../../priv/lexicons/smoke", __DIR__)]) - on_exit(fn -> Application.put_env(:tempest, Registry, previous_config) end) + :ok + end + + test "loads bundled generated Lexicon documents by default" do + Application.put_env(:tempest, Registry, bundled?: true, paths: []) + + assert :ok = Registry.validate_startup!() + assert {:ok, document} = Registry.fetch("app.bsky.actor.profile") + assert document["id"] == "app.bsky.actor.profile" + + assert {:ok, [manifest]} = Registry.manifest() + assert manifest["source_commit"] == "smoke-fixture" + assert manifest["document_count"] == 4 + + assert manifest["document_ids"] == [ + "app.bsky.actor.profile", + "com.atproto.label.defs", + "com.atproto.lexicon.schema", + "com.atproto.repo.strongRef" + ] + end + + test "loads Lexicon documents from configured fixture paths" do + Application.put_env(:tempest, Registry, + bundled?: false, + paths: [Path.expand("../../../priv/lexicons/smoke", __DIR__)] + ) + assert {:ok, document} = Registry.fetch("app.bsky.actor.profile") assert document["id"] == "app.bsky.actor.profile" assert {:ok, _document, %{"type" => "object"}} = Registry.fetch_definition("com.atproto.repo.strongRef") end + + test "configured local documents are validated and can add custom record schemas" do + directory = tmp_dir!("custom-lexicons") + + custom = + lexicon("example.app.note", %{ + "main" => %{ + "type" => "record", + "key" => "any", + "record" => %{ + "type" => "object", + "required" => ["text"], + "properties" => %{"text" => %{"type" => "string", "maxLength" => 64}} + } + } + }) + + write_json!(Path.join(directory, "example.app.note.json"), custom) + + Application.put_env(:tempest, Registry, bundled?: false, paths: [directory]) + + assert :ok = Registry.validate_startup!() + assert {:ok, _document, %{"type" => "record"}} = Registry.fetch_record("example.app.note") + end + + test "duplicate document ids fail validation" do + document = lexicon("example.app.duplicate", %{"main" => %{"type" => "object"}}) + + assert {:error, {:duplicate_document_ids, ["example.app.duplicate"]}} = + Document.validate_documents([document, document]) + end + + test "duplicate union refs fail validation" do + document = + lexicon("example.app.union", %{ + "main" => %{ + "type" => "object", + "properties" => %{ + "subject" => %{ + "type" => "union", + "refs" => ["#item", "example.app.union#item"] + } + } + }, + "item" => %{"type" => "object"} + }) + + assert {:error, {:duplicate_refs, "example.app.union#main.properties.subject", ["example.app.union#item"]}} = + Document.validate_document(document) + end + + test "loader limits fail configured local directories" do + directory = tmp_dir!("limited-lexicons") + write_json!(Path.join(directory, "one.json"), lexicon("example.app.one", %{"main" => %{"type" => "object"}})) + write_json!(Path.join(directory, "two.json"), lexicon("example.app.two", %{"main" => %{"type" => "object"}})) + + assert {:error, {:loader_limit_exceeded, :max_files}} = + Registry.validate_config(bundled?: false, paths: [directory], limits: [max_files: 1]) + end + + test "unresolved refs fail startup validation" do + document = + lexicon("example.app.unresolved", %{ + "main" => %{ + "type" => "object", + "properties" => %{"missing" => %{"type" => "ref", "ref" => "example.app.missing"}} + } + }) + + assert {:error, {:unresolved_definition_refs, ["example.app.missing#main"]}} = + Registry.validate_config(bundled?: false, documents: [document]) + end + + test "ref cycles fail document set validation" do + document = + lexicon("example.app.cycle", %{ + "main" => %{"type" => "object", "properties" => %{"next" => %{"type" => "ref", "ref" => "#node"}}}, + "node" => %{"type" => "object", "properties" => %{"next" => %{"type" => "ref", "ref" => "#node"}}} + }) + + assert {:error, {:ref_cycle, ["example.app.cycle#node", "example.app.cycle#node"]}} = + Document.validate_documents([document]) + end + + test "deep refs fail document set validation when they exceed loader limits" do + defs = + 0..4 + |> Enum.map(fn index -> + name = "node#{index}" + next = "node#{index + 1}" + + definition = + if index == 4 do + %{"type" => "object"} + else + %{"type" => "object", "properties" => %{"next" => %{"type" => "ref", "ref" => "##{next}"}}} + end + + {name, definition} + end) + |> Map.new() + + document = lexicon("example.app.deep", defs) + + assert {:error, {:loader_limit_exceeded, :max_ref_depth, path}} = + Document.validate_documents([document], max_ref_depth: 2) + + assert "example.app.deep#node3" in path + end + + test "oversized local schemas fail loader validation" do + directory = tmp_dir!("oversized-lexicons") + path = Path.join(directory, "large.json") + + write_json!( + path, + lexicon("example.app.large", %{ + "main" => %{"type" => "object", "description" => String.duplicate("x", 128)} + }) + ) + + assert {:error, {:loader_limit_exceeded, :max_file_bytes, ^path}} = + Registry.validate_config(bundled?: false, paths: [directory], limits: [max_file_bytes: 32]) + end + + test "external resolver is disabled by default" do + Application.put_env(:tempest, Registry, + bundled?: false, + paths: [], + external_resolver: [ + resolver: Tempest.Lexicon.RegistryTest.Resolver, + opts: [caller: self(), document: custom_record("example.app.external")] + ] + ) + + assert {:error, :unknown_lexicon} = Registry.fetch_record("example.app.external") + refute_received {:resolved, "example.app.external"} + end + + test "external resolver can resolve unknown schemas when explicitly enabled" do + Application.put_env(:tempest, Registry, + bundled?: false, + paths: [], + external_resolver: [ + enabled?: true, + resolver: Tempest.Lexicon.RegistryTest.Resolver, + opts: [caller: self(), document: custom_record("example.app.external")] + ] + ) + + assert {:ok, _document, %{"type" => "record"}} = Registry.fetch_record("example.app.external") + assert_received {:resolved, "example.app.external"} + end + + test "external resolver cannot override bundled or local sources" do + Application.put_env(:tempest, Registry, + bundled?: true, + paths: [], + external_resolver: [ + enabled?: true, + resolver: Tempest.Lexicon.RegistryTest.Resolver, + opts: [caller: self(), document: custom_record("app.bsky.actor.profile")] + ] + ) + + assert {:ok, document, _definition} = Registry.fetch_record("app.bsky.actor.profile") + assert document["defs"]["main"]["key"] == "literal:self" + refute_received {:resolved, "app.bsky.actor.profile"} + end + + defp lexicon(id, defs), do: %{"lexicon" => 1, "id" => id, "defs" => defs} + + defp custom_record(id) do + lexicon(id, %{ + "main" => %{ + "type" => "record", + "key" => "any", + "record" => %{"type" => "object", "properties" => %{"text" => %{"type" => "string"}}} + } + }) + end + + defp tmp_dir!(name) do + path = Path.join(System.tmp_dir!(), "tempest-#{name}-#{System.unique_integer([:positive])}") + File.mkdir_p!(path) + path + end + + defp write_json!(path, data), do: File.write!(path, Jason.encode!(data)) +end + +defmodule Tempest.Lexicon.RegistryTest.Resolver do + @moduledoc false + + @behaviour Tempest.Lexicon.ExternalResolver + + @impl true + def resolve(id, opts) do + send(Keyword.fetch!(opts, :caller), {:resolved, id}) + + case Keyword.fetch!(opts, :document) do + %{"id" => ^id} = document -> {:ok, document} + _document -> {:error, :not_found} + end + end end diff --git a/test/tempest/lexicon/validator_test.exs b/test/tempest/lexicon/validator_test.exs index a475911..0e28f1d 100644 --- a/test/tempest/lexicon/validator_test.exs +++ b/test/tempest/lexicon/validator_test.exs @@ -55,4 +55,61 @@ defmodule Tempest.Lexicon.ValidatorTest do assert {:error, :unknown_lexicon} = Validator.validate_record("example.app.record", "abc", record, require_schema?: true) end + + test "preserves record write validation modes" do + assert {:ok, :valid} = + Validator.validate_record("app.bsky.actor.profile", "self", %{ + "$type" => "app.bsky.actor.profile", + "displayName" => "Alice" + }) + + assert {:ok, :unknown} = + Validator.validate_record("example.app.record", "abc", %{"$type" => "example.app.record"}) + + assert {:error, :unknown_lexicon} = + Validator.validate_record("example.app.record", "abc", %{"$type" => "example.app.record"}, + require_schema?: true + ) + + assert {:ok, :unknown} = + Validator.validate_record( + "app.bsky.actor.profile", + "not-self", + %{"$type" => "app.bsky.actor.profile", "displayName" => 123}, + validate_schema?: false + ) + end + + test "external resolver failures preserve optimistic unknown and strict unknown failure" do + previous_config = Application.get_env(:tempest, Tempest.Lexicon.Registry, []) + + Application.put_env(:tempest, Tempest.Lexicon.Registry, + bundled?: false, + paths: [], + external_resolver: [ + enabled?: true, + resolver: Tempest.Lexicon.ValidatorTest.FailingResolver + ] + ) + + on_exit(fn -> + Application.put_env(:tempest, Tempest.Lexicon.Registry, previous_config) + end) + + record = %{"$type" => "example.app.record"} + + assert {:ok, :unknown} = Validator.validate_record("example.app.record", "abc", record) + + assert {:error, :unknown_lexicon} = + Validator.validate_record("example.app.record", "abc", record, require_schema?: true) + end +end + +defmodule Tempest.Lexicon.ValidatorTest.FailingResolver do + @moduledoc false + + @behaviour Tempest.Lexicon.ExternalResolver + + @impl true + def resolve(_id, _opts), do: {:error, :private_ip} end