diff --git a/Makefile b/Makefile index dddb783..c550105 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ CSS_SRC := $(sort $(wildcard web/css/*.css)) LEXICON_CHECK_REF ?= origin/main .PHONY: help gen css build dev run test format check clean docker-build up down logs \ - lexicon-check \ + lexicon-check lexicon-vendor \ e2e-setup e2e-up e2e-down e2e help: ## list available targets @@ -65,6 +65,9 @@ lexicon-check: ## fail if any dev.mokkenstorm.* lexicon has a BREAKING change vs git archive "$$ref" -- lexicons | tar -x -C "$$base_dir"; \ cd shared && gleam run -m tools/lexicon_check -- ../lexicons "$$base_dir/lexicons" +lexicon-vendor: ## re-resolve vendored third-party lexicons (com.atproto.*, blue.microcosm.*) and refresh lexicons.lock.json (network access; never run in CI) + cd shared && gleam run -m tools/lexicon_vendor -- ../lexicons "$$(date -u +%Y-%m-%dT%H:%M:%SZ)" + clean: ## remove build artifacts and generated codecs rm -rf build shared/build server/build web/build web/dist rm -rf shared/src/at_record/gen diff --git a/lexicons/lexicons.lock.json b/lexicons/lexicons.lock.json new file mode 100644 index 0000000..770af35 --- /dev/null +++ b/lexicons/lexicons.lock.json @@ -0,0 +1,60 @@ +{ + "version": 1, + "entries": [ + { + "nsid": "blue.microcosm.identity.resolveMiniDoc", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for microcosm.blue" + }, + { + "nsid": "blue.microcosm.links.getBacklinks", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for microcosm.blue" + }, + { + "nsid": "com.atproto.repo.createRecord", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.defs", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.deleteRecord", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.getRecord", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.listRecords", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.putRecord", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.strongRef", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.repo.uploadBlob", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + }, + { + "nsid": "com.atproto.sync.getBlob", + "status": "hand-vendored", + "reason": "no _lexicon TXT record published for atproto.com" + } + ] +} diff --git a/shared/gleam.toml b/shared/gleam.toml index 1cb5d86..9948400 100644 --- a/shared/gleam.toml +++ b/shared/gleam.toml @@ -15,3 +15,5 @@ atproto_sdl = ">= 0.1.0 and < 1.0.0" atproto_lexicon = ">= 0.1.0 and < 1.0.0" argv = ">= 1.1.0 and < 2.0.0" simplifile = ">= 2.4.0 and < 3.0.0" +gleam_http = ">= 4.3.0 and < 5.0.0" +gleam_httpc = ">= 5.0.0 and < 6.0.0" diff --git a/shared/manifest.toml b/shared/manifest.toml index d0f00ba..e849a6a 100644 --- a/shared/manifest.toml +++ b/shared/manifest.toml @@ -17,6 +17,7 @@ packages = [ { name = "gleam_crypto", version = "1.6.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_crypto", source = "hex", outer_checksum = "2DE9E4EF53CF6FEE049D4F765731F7178F7A11AEFAE00EEE63BF7536B354AD3F" }, { name = "gleam_erlang", version = "1.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_erlang", source = "hex", outer_checksum = "1124AD3AA21143E5AF0FC5CF3D9529F6DB8CA03E43A55711B60B6B7B3874375C" }, { name = "gleam_http", version = "4.3.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_http", source = "hex", outer_checksum = "82EA6A717C842456188C190AFB372665EA56CE13D8559BF3B1DD9E40F619EE0C" }, + { name = "gleam_httpc", version = "5.0.0", build_tools = ["gleam"], requirements = ["gleam_erlang", "gleam_http", "gleam_stdlib"], otp_app = "gleam_httpc", source = "hex", outer_checksum = "C545172618D07811494E97AAA4A0FB34DA6F6D0061FDC8041C2F8E3BE2B2E48F" }, { name = "gleam_json", version = "3.1.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_json", source = "hex", outer_checksum = "44FDAA8847BE8FC48CA7A1C089706BD54BADCC4C45B237A992EDDF9F2CDB2836" }, { name = "gleam_stdlib", version = "1.0.3", build_tools = ["gleam"], requirements = [], otp_app = "gleam_stdlib", source = "hex", outer_checksum = "1F543AFBA5D33DA493E6087F4E4C4F20D899411343512686C98A8ABB2963CF22" }, { name = "gleam_time", version = "1.8.0", build_tools = ["gleam"], requirements = ["gleam_stdlib"], otp_app = "gleam_time", source = "hex", outer_checksum = "533D8723774D61AD4998324F5DD1DABDCDBFABAFB9E87CB5D03C6955448FC97D" }, @@ -33,6 +34,8 @@ atproto_client = { version = ">= 0.1.0 and < 1.0.0" } atproto_codegen = { version = ">= 0.2.0 and < 1.0.0" } atproto_lexicon = { version = ">= 0.1.0 and < 1.0.0" } atproto_sdl = { version = ">= 0.1.0 and < 1.0.0" } +gleam_http = { version = ">= 4.3.0 and < 5.0.0" } +gleam_httpc = { version = ">= 5.0.0 and < 6.0.0" } gleam_json = { version = ">= 3.1.0 and < 4.0.0" } gleam_stdlib = { version = ">= 1.0.0 and < 2.0.0" } gleeunit = { version = ">= 1.0.0 and < 2.0.0" } diff --git a/shared/test/tools/lexicon_vendor.gleam b/shared/test/tools/lexicon_vendor.gleam new file mode 100644 index 0000000..c376eeb --- /dev/null +++ b/shared/test/tools/lexicon_vendor.gleam @@ -0,0 +1,229 @@ +//// Dev-only CLI: refreshes `lexicons/lexicons.lock.json` by resolving the +//// vendored third-party lexicon families (`com.atproto.*`, +//// `blue.microcosm.*`) over the network via `atproto_lexicon/source`'s NSID +//// authority resolver. A family that resolves gets its SDL regenerated from +//// the fetched schema (via `atproto_sdl.print`, the same printer the +//// `--to sdl` CLI mode uses) and a "resolved" lock entry with full +//// provenance. A family whose authority isn't network-published (as of +//// writing, neither atproto.com nor microcosm.blue publish a `_lexicon` DNS +//// TXT record) keeps its existing hand-vendored SDL untouched and gets a +//// "hand-vendored" lock entry recording why, so re-running this tool is +//// always safe: it never overwrites a file it couldn't actually re-fetch. +//// +//// Wired as `make lexicon-vendor`. This is the only place in the toolchain +//// that touches the network: `make gen` and `make test` stay offline. + +import argv +import atproto_lexicon/source.{ + type FetchedRecord, type NetworkConfig, NsidWildcard, +} +import atproto_sdl +import gleam/http/request.{type Request} +import gleam/http/response.{type Response} +import gleam/httpc +import gleam/int +import gleam/io +import gleam/json +import gleam/list +import gleam/result +import gleam/string +import simplifile + +const usage = "usage: gleam run -m tools/lexicon_vendor -- " + +/// The vendored third-party lexicon families, one NSID wildcard prefix per +/// hand-vendored directory under lexicons/. +const families = ["com.atproto", "blue.microcosm"] + +const max_tries = 3 + +pub fn main() -> Nil { + case argv.load().arguments { + [lexicons_dir, fetched_at] -> run(lexicons_dir, fetched_at) + _ -> { + io.println_error(usage) + halt(1) + } + } +} + +pub type Entry { + Resolved( + nsid: String, + did: String, + cid: String, + fetched_at: String, + source: String, + ) + HandVendored(nsid: String, reason: String) +} + +fn run(lexicons_dir: String, fetched_at: String) -> Nil { + let network = source.default_network_config() + let entries = + families + |> list.flat_map(process_family(lexicons_dir, _, network, fetched_at)) + |> list.sort(fn(a, b) { string.compare(entry_nsid(a), entry_nsid(b)) }) + + let lock_path = lexicons_dir <> "/lexicons.lock.json" + case write_lock(lock_path, entries) { + Ok(_) -> io.println("wrote " <> lock_path) + Error(reason) -> { + io.println_error(reason) + halt(1) + } + } +} + +fn process_family( + lexicons_dir: String, + prefix: String, + network: NetworkConfig, + fetched_at: String, +) -> List(Entry) { + case source.resolve(send, NsidWildcard(prefix), network) { + Ok(records) -> { + io.println( + prefix + <> ".*: resolved " + <> int.to_string(list.length(records)) + <> " schema(s) over the network", + ) + list.map(records, fn(record) { + write_sdl(lexicons_dir, record) + Resolved( + nsid: record.nsid, + did: record.did, + cid: record.cid, + fetched_at:, + source: prefix <> ".*", + ) + }) + } + Error(e) -> { + let reason = source.describe(e) + io.println( + prefix <> ".*: resolve failed (" <> reason <> "), keeping hand-vendored", + ) + hand_vendored_entries(lexicons_dir, prefix, reason) + } + } +} + +fn write_sdl(lexicons_dir: String, record: FetchedRecord) -> Nil { + let path = + lexicons_dir <> "/" <> string.replace(record.nsid, ".", "/") <> ".sdl" + let assert Ok(_) = simplifile.create_directory_all(parent_dir(path)) + let assert Ok(_) = simplifile.write(path, atproto_sdl.print(record.doc)) + Nil +} + +/// The resolver couldn't confirm this family over the network; report the +/// NSIDs it currently vendors by hand (walking the existing SDL tree) rather +/// than silently dropping them from the lockfile. +fn hand_vendored_entries( + lexicons_dir: String, + prefix: String, + reason: String, +) -> List(Entry) { + let dir = lexicons_dir <> "/" <> string.replace(prefix, ".", "/") + case simplifile.get_files(dir) { + Ok(paths) -> + paths + |> list.filter(string.ends_with(_, ".sdl")) + |> list.map(fn(path) { + HandVendored(nsid: nsid_of(lexicons_dir, path), reason:) + }) + Error(_) -> [] + } +} + +fn nsid_of(lexicons_dir: String, path: String) -> String { + let prefix = string.remove_suffix(lexicons_dir, "/") <> "/" + path + |> string.remove_prefix(prefix) + |> string.remove_suffix(".sdl") + |> string.replace("/", ".") +} + +fn parent_dir(path: String) -> String { + case path |> string.split("/") |> list.reverse { + [] | [_] -> "." + [_, ..rest] -> rest |> list.reverse |> string.join("/") + } +} + +fn entry_nsid(entry: Entry) -> String { + case entry { + Resolved(nsid:, ..) -> nsid + HandVendored(nsid:, ..) -> nsid + } +} + +/// Hand-indented (not `json.to_string`, which is a single line) so +/// `git diff` on a re-vendor shows exactly the entries that changed. +fn write_lock(lock_path: String, entries: List(Entry)) -> Result(Nil, String) { + simplifile.write(lock_path, encode_lock(entries)) + |> result.map_error(fn(e) { + lock_path <> ": write failed: " <> string.inspect(e) + }) +} + +fn encode_lock(entries: List(Entry)) -> String { + let items = entries |> list.map(encode_entry) |> string.join(",\n") + "{\n \"version\": 1,\n \"entries\": [\n" <> items <> "\n ]\n}\n" +} + +fn encode_entry(entry: Entry) -> String { + case entry { + Resolved(nsid:, did:, cid:, fetched_at:, source:) -> + json_object([ + #("nsid", nsid), + #("status", "resolved"), + #("did", did), + #("cid", cid), + #("fetchedAt", fetched_at), + #("source", source), + ]) + HandVendored(nsid:, reason:) -> + json_object([ + #("nsid", nsid), + #("status", "hand-vendored"), + #("reason", reason), + ]) + } +} + +fn json_object(fields: List(#(String, String))) -> String { + let body = + fields + |> list.map(fn(f) { " \"" <> f.0 <> "\": " <> json_string(f.1) }) + |> string.join(",\n") + " {\n" <> body <> "\n }" +} + +fn json_string(value: String) -> String { + json.string(value) |> json.to_string +} + +fn send(req: Request(BitArray)) -> Result(Response(BitArray), String) { + send_with_retry(req, max_tries) +} + +fn send_with_retry( + req: Request(BitArray), + tries: Int, +) -> Result(Response(BitArray), String) { + case + httpc.configure() + |> httpc.follow_redirects(True) + |> httpc.dispatch_bits(req) + { + Ok(resp) -> Ok(resp) + Error(_) if tries > 1 -> send_with_retry(req, tries - 1) + Error(e) -> Error(string.inspect(e)) + } +} + +@external(erlang, "erlang", "halt") +fn halt(code: Int) -> Nil