diff --git a/config/runtime.exs b/config/runtime.exs index e0f22fc..1a2eb92 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -38,6 +38,10 @@ if blob_max_bytes = System.get_env("TEMPEST_BLOB_MAX_BYTES") do config :tempest, Tempest.Config, blob_max_bytes: String.to_integer(blob_max_bytes) end +if blob_cdn_base_url = System.get_env("TEMPEST_BLOB_CDN_BASE_URL") do + config :tempest, Tempest.Blobs, cdn_base_url: blob_cdn_base_url +end + if config_env() == :prod do # The secret key base is used to sign/encrypt cookies and other secrets. # A default value is used in config/dev.exs and config/test.exs but you diff --git a/docs/tasks/08-blobs.md b/docs/tasks/08-blobs.md index 2525eb6..f94b58c 100644 --- a/docs/tasks/08-blobs.md +++ b/docs/tasks/08-blobs.md @@ -21,10 +21,10 @@ Goal: upload, reference, serve, and garbage collect blobs. - [x] T08-11: Implement `getBlob`. - [x] T08-12: Suppress blob serving for inactive accounts. - [x] T08-13: Add blob garbage collector. -- [ ] T08-14: Add integration tests for upload, reference, get, delete. +- [x] T08-14: Add integration tests for upload, reference, get, delete. - [x] T08-15: Add CSP and nosniff headers to `getBlob`. -- [ ] T08-16: Add S3-compatible storage behavior and local adapter contract tests. -- [ ] T08-17: Add optional CDN redirect behavior with inactive-account suppression. +- [x] T08-16: Add S3-compatible storage behavior and local adapter contract tests. +- [x] T08-17: Add optional CDN redirect behavior with inactive-account suppression. ## Integration Tests diff --git a/lib/tempest/blobs.ex b/lib/tempest/blobs.ex index 07c331c..90a2a68 100644 --- a/lib/tempest/blobs.ex +++ b/lib/tempest/blobs.ex @@ -232,6 +232,21 @@ defmodule Tempest.Blobs do end) end + @doc """ + Deletes metadata rows for blobs that are no longer referenced. + """ + @spec delete_metadata(String.t(), [String.t()]) :: :ok | {:error, term()} + def delete_metadata(_did, []), do: :ok + + def delete_metadata(did, cids) when is_binary(did) and is_list(cids) do + Enum.reduce_while(Enum.uniq(cids), :ok, fn cid, :ok -> + case Repo.query("DELETE FROM blob_metadata WHERE did = ?1 AND cid = ?2", [did, cid]) do + {:ok, _result} -> {:cont, :ok} + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + @doc """ Extracts valid AT Protocol blob reference CIDs from a record-shaped value. """ @@ -243,6 +258,20 @@ defmodule Tempest.Blobs do |> Enum.sort() end + @doc """ + Returns a CDN URL for a public blob when CDN redirects are configured. + """ + @spec cdn_url(String.t(), String.t()) :: {:ok, String.t()} | :disabled + def cdn_url(did, cid) when is_binary(did) and is_binary(cid) do + case Keyword.get(blob_config(), :cdn_base_url) do + base_url when is_binary(base_url) and base_url != "" -> + {:ok, base_url |> String.trim_trailing("/") |> Kernel.<>("/blobs/" <> encode_path_segment(did) <> "/" <> cid)} + + _disabled -> + :disabled + end + end + @doc """ Validates the declared content length against the actual byte size and limit. """ @@ -363,4 +392,10 @@ defmodule Tempest.Blobs do |> Map.values() |> Enum.reduce(acc, &collect_blob_cids/2) end + + defp blob_config do + Application.get_env(:tempest, __MODULE__, []) + end + + defp encode_path_segment(value), do: URI.encode(value, &URI.char_unreserved?/1) end diff --git a/lib/tempest/blobs/local_storage.ex b/lib/tempest/blobs/local_storage.ex index 62824b4..8c0da01 100644 --- a/lib/tempest/blobs/local_storage.ex +++ b/lib/tempest/blobs/local_storage.ex @@ -3,6 +3,8 @@ defmodule Tempest.Blobs.LocalStorage do Local filesystem blob storage adapter. """ + @behaviour Tempest.Blobs.StorageAdapter + alias Tempest.Config alias Tempest.RepoCore.{Cid, Did} diff --git a/lib/tempest/blobs/s3_storage.ex b/lib/tempest/blobs/s3_storage.ex new file mode 100644 index 0000000..3c89334 --- /dev/null +++ b/lib/tempest/blobs/s3_storage.ex @@ -0,0 +1,157 @@ +defmodule Tempest.Blobs.S3Storage do + @moduledoc """ + S3-compatible object storage adapter. + + This adapter assumes the configured endpoint accepts the supplied request + options, including any authentication headers or Req options needed by the + deployment. Metadata stays in `account.sqlite`. + """ + + @behaviour Tempest.Blobs.StorageAdapter + + alias Tempest.RepoCore.{Cid, Did} + + @impl true + def put_temp_blob(config, did, cid, bytes) when is_list(config) and is_binary(bytes) do + with {:ok, did} <- normalize_did(did), + :ok <- validate_cid(cid), + {:ok, request} <- request_options(config, temp_key(did, cid), method: :put, body: bytes) do + case Req.request(request) do + {:ok, %{status: status}} when status in 200..299 -> + {:ok, %{cid: cid, path: temp_key(did, cid), size: byte_size(bytes)}} + + {:ok, %{status: status}} -> + {:error, {:s3_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + end + + @impl true + def promote_blob(config, did, cid) when is_list(config) do + with {:ok, did} <- normalize_did(did), + :ok <- validate_cid(cid), + source <- temp_key(did, cid), + destination <- blob_key(did, cid), + {:ok, request} <- + request_options(config, destination, + method: :put, + headers: [{"x-amz-copy-source", "/" <> bucket!(config) <> "/" <> source}] + ) do + case Req.request(request) do + {:ok, %{status: status}} when status in 200..299 -> + _ = delete_temp_blob(config, did, cid) + {:ok, destination} + + {:ok, %{status: 404}} -> + {:error, :blob_not_found} + + {:ok, %{status: status}} -> + {:error, {:s3_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + end + + @impl true + def get_blob(config, did, cid, mime_type \\ "application/octet-stream") when is_list(config) do + with {:ok, did} <- normalize_did(did), + :ok <- validate_cid(cid), + {:ok, request} <- request_options(config, blob_key(did, cid), method: :get) do + case Req.request(request) do + {:ok, %{status: status, body: bytes}} when status in 200..299 and is_binary(bytes) -> + {:ok, %{bytes: bytes, content_length: byte_size(bytes), mime_type: mime_type}} + + {:ok, %{status: 404}} -> + {:error, :blob_not_found} + + {:ok, %{status: status}} -> + {:error, {:s3_status, status}} + + {:error, reason} -> + {:error, reason} + end + end + end + + @impl true + def delete_blob(config, did, cid) when is_list(config) do + with :ok <- delete_temp_blob(config, did, cid) do + delete_key(config, did, cid, &blob_key/2) + end + end + + @impl true + def delete_temp_blob(config, did, cid) when is_list(config) do + delete_key(config, did, cid, &temp_key/2) + end + + @impl true + def list_blobs(_config, _did, _opts \\ []) do + {:error, :metadata_authoritative} + end + + defp delete_key(config, did, cid, key_fun) do + with {:ok, did} <- normalize_did(did), + :ok <- validate_cid(cid), + {:ok, request} <- request_options(config, key_fun.(did, cid), method: :delete) do + case Req.request(request) do + {:ok, %{status: status}} when status in 200..299 or status == 404 -> :ok + {:ok, %{status: status}} -> {:error, {:s3_status, status}} + {:error, reason} -> {:error, reason} + end + end + end + + defp request_options(config, key, opts) do + endpoint_url = required!(config, :endpoint_url) + bucket = bucket!(config) + + request = + config + |> Keyword.get(:req_options, []) + |> Keyword.merge(opts) + |> Keyword.update(:headers, default_headers(config), &(default_headers(config) ++ List.wrap(&1))) + |> Keyword.put(:url, object_url(endpoint_url, bucket, key)) + + {:ok, request} + rescue + e in KeyError -> {:error, {:missing_s3_config, e.key}} + end + + defp object_url(endpoint_url, bucket, key) do + endpoint_url + |> String.trim_trailing("/") + |> Kernel.<>("/" <> URI.encode(bucket) <> "/" <> encode_key(key)) + end + + defp encode_key(key) do + key + |> String.split("/") + |> Enum.map(fn segment -> URI.encode(segment, &URI.char_unreserved?/1) end) + |> Enum.join("/") + end + + defp default_headers(config), do: Keyword.get(config, :headers, []) + + defp required!(config, key), do: Keyword.fetch!(config, key) + defp bucket!(config), do: required!(config, :bucket) + + defp temp_key(did, cid), do: "temp/blobs/" <> did <> "/" <> cid + defp blob_key(did, cid), do: "blobs/" <> did <> "/" <> cid + + defp normalize_did(did) do + case Did.parse(did) do + {:ok, did} -> {:ok, did} + {:error, _reason} -> {:error, :invalid_did} + end + end + + defp validate_cid(cid) do + if Cid.valid?(cid), do: :ok, else: {:error, :invalid_cid} + end +end diff --git a/lib/tempest/blobs/storage_adapter.ex b/lib/tempest/blobs/storage_adapter.ex new file mode 100644 index 0000000..3bfc3d9 --- /dev/null +++ b/lib/tempest/blobs/storage_adapter.ex @@ -0,0 +1,24 @@ +defmodule Tempest.Blobs.StorageAdapter do + @moduledoc """ + Blob storage adapter contract. + + Local metadata remains authoritative; adapters only own bytes. + """ + + @type config :: term() + @type blob_read :: %{bytes: binary(), content_length: non_neg_integer(), mime_type: String.t()} + + @callback put_temp_blob(config(), String.t(), String.t(), binary()) :: + {:ok, %{cid: String.t(), path: String.t(), size: non_neg_integer()}} | {:error, term()} + + @callback promote_blob(config(), String.t(), String.t()) :: {:ok, String.t()} | {:error, term()} + + @callback get_blob(config(), String.t(), String.t(), String.t()) :: {:ok, blob_read()} | {:error, term()} + + @callback delete_blob(config(), String.t(), String.t()) :: :ok | {:error, term()} + + @callback delete_temp_blob(config(), String.t(), String.t()) :: :ok | {:error, term()} + + @callback list_blobs(config(), String.t(), keyword()) :: + {:ok, %{required(:cids) => [String.t()], optional(:cursor) => String.t()}} | {:error, term()} +end diff --git a/lib/tempest/records.ex b/lib/tempest/records.ex index 57ab137..e21f752 100644 --- a/lib/tempest/records.ex +++ b/lib/tempest/records.ex @@ -57,6 +57,7 @@ defmodule Tempest.Records do LexiconValidator.validate_record(input.collection, input.rkey, input.record, input.validate), blob_cids = Blobs.referenced_cids(input.record), :ok <- Blobs.ensure_present(account.did, blob_cids), + {:ok, old_blob_cids} <- current_record_blob_cids(account.did, input.collection, input.rkey), {:ok, signing_key} <- active_signing_key(account), {:ok, stored} <- RepoStorage.put_record(account, signing_key, %{ @@ -67,6 +68,7 @@ defmodule Tempest.Records do swap_commit: input.swap_commit }), :ok <- promote_blobs(account.did, blob_cids), + :ok <- delete_unreferenced_blobs(account.did, old_blob_cids), {:ok, _event} <- insert_sequence_event(account.did, stored, "update", input.collection, input.rkey) do {:ok, %{ @@ -84,6 +86,7 @@ defmodule Tempest.Records do def delete_record(%AuthContext{account: account}, params) do with {:ok, input} <- LexiconValidator.validate_delete_record_input(params), :ok <- ensure_repo_owner(account, input.repo), + {:ok, old_blob_cids} <- current_record_blob_cids(account.did, input.collection, input.rkey), {:ok, signing_key} <- active_signing_key(account), {:ok, stored} <- RepoStorage.delete_record(account, signing_key, %{ @@ -92,6 +95,7 @@ defmodule Tempest.Records do swap_record: input.swap_record, swap_commit: input.swap_commit }), + :ok <- maybe_delete_unreferenced_blobs(account.did, old_blob_cids, stored), {:ok, _event} <- maybe_insert_delete_event(account.did, stored, input.collection, input.rkey) do if stored.deleted? do {:ok, @@ -212,6 +216,50 @@ defmodule Tempest.Records do end) end + defp current_record_blob_cids(did, collection, rkey) do + case RepoStorage.get_record(did, collection, rkey) do + {:ok, %{value: record}} -> {:ok, Blobs.referenced_cids(record)} + {:error, :record_not_found} -> {:ok, []} + {:error, reason} -> {:error, reason} + end + end + + defp maybe_delete_unreferenced_blobs(did, old_blob_cids, %{deleted?: true}) do + delete_unreferenced_blobs(did, old_blob_cids) + end + + defp maybe_delete_unreferenced_blobs(_did, _old_blob_cids, %{deleted?: false}), do: :ok + + defp delete_unreferenced_blobs(_did, []), do: :ok + + defp delete_unreferenced_blobs(did, old_blob_cids) do + config = Config.load!() + + with {:ok, current_blob_cids} <- all_current_blob_cids(did) do + old_blob_cids + |> Enum.reject(&(&1 in current_blob_cids)) + |> Enum.reduce_while(:ok, fn cid, :ok -> + with :ok <- LocalStorage.delete_blob(config, did, cid), + :ok <- Blobs.delete_metadata(did, [cid]) do + {:cont, :ok} + else + {:error, reason} -> {:halt, {:error, reason}} + end + end) + end + end + + defp all_current_blob_cids(did, cursor \\ nil, acc \\ []) do + with {:ok, page} <- RepoStorage.list_referenced_blobs(did, limit: 1000, cursor: cursor) do + cids = acc ++ page.cids + + case Map.get(page, :cursor) do + nil -> {:ok, cids} + cursor -> all_current_blob_cids(did, cursor, cids) + end + end + end + defp insert_sequence_event(did, stored, action, collection, rkey) do path = collection <> "/" <> rkey diff --git a/lib/tempest/sync.ex b/lib/tempest/sync.ex index e183f1e..e444921 100644 --- a/lib/tempest/sync.ex +++ b/lib/tempest/sync.ex @@ -113,9 +113,8 @@ defmodule Tempest.Sync do {:ok, account} <- fetch_account(input.did), :ok <- ensure_active(account), {:ok, metadata} <- Blobs.get_public_metadata(account.did, input.cid), - {:ok, blob} <- LocalStorage.get_blob(Config.load!(), account.did, input.cid, metadata.mime_type), - :ok <- ensure_blob_size(blob, metadata) do - {:ok, Map.put(blob, :cid, input.cid)} + {:ok, response} <- blob_response(account.did, input.cid, metadata) do + {:ok, response} end end @@ -260,6 +259,19 @@ defmodule Tempest.Sync do defp ensure_blob_size(%{content_length: size}, %{size: size}), do: :ok defp ensure_blob_size(_blob, _metadata), do: {:error, :blob_not_found} + defp blob_response(did, cid, metadata) do + case Blobs.cdn_url(did, cid) do + {:ok, url} -> + {:ok, %{redirect: url, cid: cid}} + + :disabled -> + with {:ok, blob} <- LocalStorage.get_blob(Config.load!(), did, cid, metadata.mime_type), + :ok <- ensure_blob_size(blob, metadata) do + {:ok, Map.put(blob, :cid, cid)} + end + end + end + defp validate_request_crawl_hostname(hostname) when is_binary(hostname) do configured = Tempest.Config.load!().hostname hostname = String.trim(hostname) diff --git a/lib/tempest_web/controllers/xrpc_controller.ex b/lib/tempest_web/controllers/xrpc_controller.ex index 1b9dda0..360dc9b 100644 --- a/lib/tempest_web/controllers/xrpc_controller.ex +++ b/lib/tempest_web/controllers/xrpc_controller.ex @@ -100,6 +100,14 @@ defmodule TempestWeb.XrpcController do defp respond(conn, %{output: @json}, body), do: json(conn, body) + defp respond(conn, %{nsid: "com.atproto.sync.getBlob"}, %{redirect: url}) do + conn + |> put_resp_header("location", url) + |> put_resp_header("content-security-policy", "default-src 'none'; sandbox") + |> put_resp_header("x-content-type-options", "nosniff") + |> send_resp(302, "") + end + defp respond(conn, %{nsid: "com.atproto.sync.getBlob"}, %{bytes: bytes} = blob) do conn |> put_resp_content_type(blob.mime_type, nil) diff --git a/test/smoke/README.md b/test/smoke/README.md index 61ad57c..c09b74f 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -45,3 +45,11 @@ hurl --test --jobs 1 \ --variable base_url=http://localhost:4000 \ test/smoke/firehose.hurl ``` + +Run the blob smoke test: + +```bash +hurl --test --jobs 1 \ + --variable base_url=http://localhost:4000 \ + test/smoke/blobs.hurl +``` diff --git a/test/smoke/blobs.hurl b/test/smoke/blobs.hurl new file mode 100644 index 0000000..44af4ed --- /dev/null +++ b/test/smoke/blobs.hurl @@ -0,0 +1,102 @@ +POST {{base_url}}/xrpc/com.atproto.server.createAccount +Content-Type: application/json +{ + "handle": "blobs-{{newUuid}}.test", + "email": "blobs-{{newUuid}}@example.com", + "password": "correct horse battery staple" +} +HTTP 200 +[Captures] +created_did: jsonpath "$.did" +access_token: jsonpath "$.accessJwt" +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.did" startsWith "did:plc:" + +POST {{base_url}}/xrpc/com.atproto.repo.uploadBlob +Authorization: Bearer {{access_token}} +Content-Type: text/plain +`hello blob` +HTTP 200 +[Captures] +blob_cid: jsonpath "$.blob.ref.$link" +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.blob.$type" == "blob" +jsonpath "$.blob.mimeType" == "text/plain" +jsonpath "$.blob.size" == 10 + +GET {{base_url}}/xrpc/com.atproto.sync.listBlobs?did={{created_did}} +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.cids[0]" not exists + +GET {{base_url}}/xrpc/com.atproto.sync.getBlob?did={{created_did}}&cid={{blob_cid}} +HTTP 400 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.error" == "BlobNotFound" + +POST {{base_url}}/xrpc/com.atproto.repo.createRecord +Authorization: Bearer {{access_token}} +Content-Type: application/json +{ + "repo": "{{created_did}}", + "collection": "app.tempest.blob", + "rkey": "avatar", + "validate": false, + "record": { + "$type": "app.tempest.blob", + "image": { + "$type": "blob", + "ref": {"$link": "{{blob_cid}}"}, + "mimeType": "text/plain", + "size": 10 + } + } +} +HTTP 200 +[Captures] +record_cid: jsonpath "$.cid" +record_commit_cid: jsonpath "$.commit.cid" +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.uri" == "at://{{created_did}}/app.tempest.blob/avatar" + +GET {{base_url}}/xrpc/com.atproto.sync.listBlobs?did={{created_did}} +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.cids[0]" == "{{blob_cid}}" +jsonpath "$.cids[1]" not exists + +GET {{base_url}}/xrpc/com.atproto.sync.getBlob?did={{created_did}}&cid={{blob_cid}} +HTTP 200 +[Asserts] +header "content-type" == "text/plain" +header "content-length" == "10" +header "x-content-type-options" == "nosniff" +header "content-security-policy" == "default-src 'none'; sandbox" +body == "hello blob" + +POST {{base_url}}/xrpc/com.atproto.repo.deleteRecord +Authorization: Bearer {{access_token}} +Content-Type: application/json +{ + "repo": "{{created_did}}", + "collection": "app.tempest.blob", + "rkey": "avatar", + "swapRecord": "{{record_cid}}", + "swapCommit": "{{record_commit_cid}}" +} +HTTP 200 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.commit.cid" exists + +GET {{base_url}}/xrpc/com.atproto.sync.getBlob?did={{created_did}}&cid={{blob_cid}} +HTTP 400 +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.error" == "BlobNotFound" diff --git a/test/smoke/car-sync.hurl b/test/smoke/car-sync.hurl index f5dfc25..4aaec96 100644 --- a/test/smoke/car-sync.hurl +++ b/test/smoke/car-sync.hurl @@ -76,6 +76,19 @@ HTTP 200 header "content-type" contains "application/json" jsonpath "$.repos" exists +POST {{base_url}}/xrpc/com.atproto.repo.uploadBlob +Authorization: Bearer {{access_token}} +Content-Type: text/plain +`hello blob` +HTTP 200 +[Captures] +blob_cid: jsonpath "$.blob.ref.$link" +[Asserts] +header "content-type" contains "application/json" +jsonpath "$.blob.$type" == "blob" +jsonpath "$.blob.mimeType" == "text/plain" +jsonpath "$.blob.size" == 10 + POST {{base_url}}/xrpc/com.atproto.repo.createRecord Authorization: Bearer {{access_token}} Content-Type: application/json @@ -88,7 +101,7 @@ Content-Type: application/json "$type": "app.tempest.blob", "image": { "$type": "blob", - "ref": {"$link": "bafkreialjtynvjdbcycvhjerlypesyvyskytxn6hlzptdwqvvqojfnukly"}, + "ref": {"$link": "{{blob_cid}}"}, "mimeType": "text/plain", "size": 10 } @@ -100,5 +113,5 @@ GET {{base_url}}/xrpc/com.atproto.sync.listBlobs?did={{created_did}} HTTP 200 [Asserts] header "content-type" contains "application/json" -jsonpath "$.cids[0]" == "bafkreialjtynvjdbcycvhjerlypesyvyskytxn6hlzptdwqvvqojfnukly" +jsonpath "$.cids[0]" == "{{blob_cid}}" jsonpath "$.cids[1]" not exists diff --git a/test/tempest/blobs/s3_storage_test.exs b/test/tempest/blobs/s3_storage_test.exs new file mode 100644 index 0000000..de98dea --- /dev/null +++ b/test/tempest/blobs/s3_storage_test.exs @@ -0,0 +1,97 @@ +defmodule Tempest.Blobs.S3StorageTest do + use ExUnit.Case, async: false + + alias Tempest.Blobs + alias Tempest.Blobs.S3Storage + + setup context do + Req.Test.set_req_test_from_context(context) + Req.Test.verify_on_exit!(context) + + config = [ + endpoint_url: "https://objects.example.test", + bucket: "tempest-test", + req_options: [plug: {Req.Test, __MODULE__}], + headers: [{"authorization", "Bearer test-token"}] + ] + + %{config: config, did: "did:plc:s3storage", cid: Blobs.cid_for("s3 bytes")} + end + + test "put_temp_blob writes to the temp object key", %{config: config, did: did, cid: cid} do + Req.Test.expect(__MODULE__, fn conn -> + assert conn.method == "PUT" + assert conn.request_path == "/tempest-test/temp/blobs/did%3Aplc%3As3storage/#{cid}" + assert Plug.Conn.get_req_header(conn, "authorization") == ["Bearer test-token"] + assert {:ok, "s3 bytes", conn} = Plug.Conn.read_body(conn) + + Plug.Conn.send_resp(conn, 200, "") + end) + + assert {:ok, stored} = S3Storage.put_temp_blob(config, did, cid, "s3 bytes") + assert stored == %{cid: cid, path: "temp/blobs/#{did}/#{cid}", size: 8} + end + + test "promote_blob copies temp object into the public blob key and deletes temp", %{ + config: config, + did: did, + cid: cid + } do + Req.Test.expect(__MODULE__, fn conn -> + assert conn.method == "PUT" + assert conn.request_path == "/tempest-test/blobs/did%3Aplc%3As3storage/#{cid}" + + assert Plug.Conn.get_req_header(conn, "x-amz-copy-source") == [ + "/tempest-test/temp/blobs/#{did}/#{cid}" + ] + + Plug.Conn.send_resp(conn, 200, "") + end) + + Req.Test.expect(__MODULE__, fn conn -> + assert conn.method == "DELETE" + assert conn.request_path == "/tempest-test/temp/blobs/did%3Aplc%3As3storage/#{cid}" + + Plug.Conn.send_resp(conn, 204, "") + end) + + assert {:ok, promoted_path} = S3Storage.promote_blob(config, did, cid) + assert promoted_path == "blobs/#{did}/#{cid}" + end + + test "get_blob reads public object bytes", %{config: config, did: did, cid: cid} do + Req.Test.expect(__MODULE__, fn conn -> + assert conn.method == "GET" + assert conn.request_path == "/tempest-test/blobs/did%3Aplc%3As3storage/#{cid}" + + conn + |> Plug.Conn.put_resp_content_type("text/plain") + |> Plug.Conn.send_resp(200, "s3 bytes") + end) + + assert {:ok, %{bytes: "s3 bytes", content_length: 8, mime_type: "text/plain"}} = + S3Storage.get_blob(config, did, cid, "text/plain") + end + + test "delete_blob deletes temp and public keys", %{config: config, did: did, cid: cid} do + Req.Test.expect(__MODULE__, fn conn -> + assert conn.method == "DELETE" + assert conn.request_path == "/tempest-test/temp/blobs/did%3Aplc%3As3storage/#{cid}" + + Plug.Conn.send_resp(conn, 404, "") + end) + + Req.Test.expect(__MODULE__, fn conn -> + assert conn.method == "DELETE" + assert conn.request_path == "/tempest-test/blobs/did%3Aplc%3As3storage/#{cid}" + + Plug.Conn.send_resp(conn, 204, "") + end) + + assert :ok = S3Storage.delete_blob(config, did, cid) + end + + test "list_blobs remains metadata-authoritative", %{config: config, did: did} do + assert {:error, :metadata_authoritative} = S3Storage.list_blobs(config, did, limit: 10) + end +end diff --git a/test/tempest/blobs/storage_adapter_contract_test.exs b/test/tempest/blobs/storage_adapter_contract_test.exs new file mode 100644 index 0000000..99863ce --- /dev/null +++ b/test/tempest/blobs/storage_adapter_contract_test.exs @@ -0,0 +1,17 @@ +defmodule Tempest.Blobs.StorageAdapterContractTest do + use ExUnit.Case, async: true + + alias Tempest.Blobs.LocalStorage + alias Tempest.Blobs.S3Storage + alias Tempest.Blobs.StorageAdapter + + test "local and S3 adapters implement the storage contract callbacks" do + for adapter <- [LocalStorage, S3Storage], + {function, arity} <- StorageAdapter.behaviour_info(:callbacks) do + Code.ensure_loaded!(adapter) + + assert function_exported?(adapter, function, arity), + "#{inspect(adapter)} must export #{function}/#{arity}" + end + end +end diff --git a/test/tempest_web/xrpc/sync_reads_test.exs b/test/tempest_web/xrpc/sync_reads_test.exs index 12f01e5..c627044 100644 --- a/test/tempest_web/xrpc/sync_reads_test.exs +++ b/test/tempest_web/xrpc/sync_reads_test.exs @@ -15,9 +15,11 @@ defmodule TempestWeb.Xrpc.SyncReadsTest do Req.Test.verify_on_exit!(context) old_sync_config = Application.get_env(:tempest, Tempest.Sync, []) + old_blob_config = Application.get_env(:tempest, Tempest.Blobs, []) on_exit(fn -> Application.put_env(:tempest, Tempest.Sync, old_sync_config) + Application.put_env(:tempest, Tempest.Blobs, old_blob_config) clear_request_crawl_rate_limits() end) @@ -284,6 +286,84 @@ defmodule TempestWeb.Xrpc.SyncReadsTest do assert %{"error" => "RepoSuspended"} = json_response(inactive_conn, 400) end + test "getBlob redirects to configured CDN only after public and active checks", %{conn: conn} do + Application.put_env(:tempest, Tempest.Blobs, cdn_base_url: "https://cdn.example.test/pds") + + account = create_account!(conn, "sync-cdn-blob.test", "sync-cdn-blob@example.com") + public_cid = upload_blob!(conn, account, "cdn public")["blob"]["ref"]["$link"] + temp_cid = upload_blob!(conn, account, "cdn temp")["blob"]["ref"]["$link"] + + create_blob_record!(conn, account, "cdn", public_cid) + + redirect_conn = + conn + |> recycle() + |> get(~p"/xrpc/com.atproto.sync.getBlob", %{"did" => account["did"], "cid" => public_cid}) + + assert redirect_conn.status == 302 + + assert get_resp_header(redirect_conn, "location") == [ + "https://cdn.example.test/pds/blobs/#{URI.encode(account["did"], &URI.char_unreserved?/1)}/#{public_cid}" + ] + + assert get_resp_header(redirect_conn, "x-content-type-options") == ["nosniff"] + + temp_conn = + conn + |> recycle() + |> get(~p"/xrpc/com.atproto.sync.getBlob", %{"did" => account["did"], "cid" => temp_cid}) + + assert %{"error" => "BlobNotFound"} = json_response(temp_conn, 400) + + Account + |> where([account], account.did == ^account["did"]) + |> Repo.update_all(set: [active: false, status: "deactivated"]) + + inactive_conn = + conn + |> recycle() + |> get(~p"/xrpc/com.atproto.sync.getBlob", %{"did" => account["did"], "cid" => public_cid}) + + assert %{"error" => "RepoDeactivated"} = json_response(inactive_conn, 400) + assert get_resp_header(inactive_conn, "location") == [] + end + + test "deleteRecord removes blob bytes when no current record references them", %{conn: conn} do + account = create_account!(conn, "sync-delete-blob.test", "sync-delete-blob@example.com") + blob = upload_blob!(conn, account, "delete blob")["blob"] + cid = blob["ref"]["$link"] + created = create_blob_record!(conn, account, "delete-me", cid) + + assert get_blob_status(conn, account["did"], cid) == 200 + + delete_conn = + conn + |> auth_json(account) + |> post(~p"/xrpc/com.atproto.repo.deleteRecord", %{ + "repo" => account["did"], + "collection" => "app.tempest.blob", + "rkey" => "delete-me", + "swapRecord" => created["cid"], + "swapCommit" => created["commit"]["cid"] + }) + + assert %{"commit" => %{"cid" => _cid}} = json_response(delete_conn, 200) + + deleted_blob_conn = + conn + |> recycle() + |> get(~p"/xrpc/com.atproto.sync.getBlob", %{"did" => account["did"], "cid" => cid}) + + assert %{"error" => "BlobNotFound"} = json_response(deleted_blob_conn, 400) + + list_conn = + conn + |> recycle() + |> get(~p"/xrpc/com.atproto.sync.listBlobs", %{"did" => account["did"]}) + + assert json_response(list_conn, 200) == %{"cids" => []} + end + test "requestCrawl fans out to configured relays", %{conn: conn} do Application.put_env(:tempest, Tempest.Sync, relays: ["https://relay.test"], @@ -478,6 +558,13 @@ defmodule TempestWeb.Xrpc.SyncReadsTest do |> json_response(200) end + defp get_blob_status(conn, did, cid) do + conn + |> recycle() + |> get(~p"/xrpc/com.atproto.sync.getBlob", %{"did" => did, "cid" => cid}) + |> Map.fetch!(:status) + end + defp upload_blob!(conn, account, bytes) do conn |> recycle()