diff --git a/crates/vibescrobble/tests/conformance/mst_vectors.rs b/crates/vibescrobble/tests/conformance/mst_vectors.rs index e4f6e4fb..0f6e81eb 100644 --- a/crates/vibescrobble/tests/conformance/mst_vectors.rs +++ b/crates/vibescrobble/tests/conformance/mst_vectors.rs @@ -36,11 +36,65 @@ const CASE_TYPE: &str = "mst-diff"; const FETCH: &str = "run scripts/fetch-mst-vectors.sh, or point MST_TEST_SUITE at a clone"; /// Where the corpus is, which is a clone and not a vendored copy. +/// +/// Cached per machine rather than per checkout. The corpus is immutable +/// upstream test data — nothing about it varies with the branch you are on — +/// and this project is worked from several worktrees at once, each of which +/// used to want its own 72MB copy and to fail four tests until somebody +/// fetched one. It sits beside the configuration and state the rest of this +/// project already keeps under the same XDG directories. +/// +/// `MST_TEST_SUITE` still wins, which is how a checkout points at a corpus of +/// its own, and how a machine with no home directory to speak of runs the +/// suite at all. +/// +/// Kept in step with `scripts/fetch-mst-vectors.sh` by +/// [`the_fetch_script_and_the_runner_agree_on_where_the_corpus_is`]. fn suite_dir() -> PathBuf { - match std::env::var_os("MST_TEST_SUITE") { - Some(path) => PathBuf::from(path), - None => PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../.cache/mst-test-suite"), + if let Some(path) = std::env::var_os("MST_TEST_SUITE") { + return PathBuf::from(path); } + cache_home().join("vibescrobble").join("mst-test-suite") +} + +/// `$XDG_CACHE_HOME`, or `~/.cache` as the specification defines. +/// +/// Falls back to the current directory, which is not useful and is not meant +/// to be: it means the corpus is reported as missing and the message names the +/// script, which is the right outcome on a machine that has neither variable. +fn cache_home() -> PathBuf { + std::env::var_os("XDG_CACHE_HOME") + .map(PathBuf::from) + .filter(|path| !path.as_os_str().is_empty()) + .or_else(|| { + std::env::var_os("HOME") + .map(PathBuf::from) + .filter(|path| !path.as_os_str().is_empty()) + .map(|home| home.join(".cache")) + }) + .unwrap_or_else(|| PathBuf::from(".")) +} + +/// The script and the runner name the same directory. +/// +/// Two places have to agree on one path and neither can import the other, so +/// the agreement is checked rather than trusted — the same shape the rest of +/// this repository uses for a constant it cannot share. +#[test] +fn the_fetch_script_and_the_runner_agree_on_where_the_corpus_is() { + let script = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../scripts/fetch-mst-vectors.sh"); + let source = std::fs::read_to_string(&script) + .unwrap_or_else(|err| panic!("cannot read {}: {err}", script.display())); + + assert!( + source.contains(r#"cache_home="${XDG_CACHE_HOME:-$HOME/.cache}""#), + "the script no longer resolves the cache directory the way the runner does" + ); + assert!( + source.contains(r#"dest="$cache_home/vibescrobble/mst-test-suite""#), + "the script no longer writes where the runner reads" + ); } /// One case file, in upstream's own shape. diff --git a/docs/conformance.md b/docs/conformance.md index a1591b7a..89375503 100644 --- a/docs/conformance.md +++ b/docs/conformance.md @@ -577,10 +577,22 @@ carry the weight either. Nothing runs this suite unattended: there is no CI here, so the only thing that runs it is a person at a terminal, and that person has a network. -So `scripts/fetch-mst-vectors.sh` clones the suite into `.cache/mst-test-suite`, -which git ignores, and the runner reads it in upstream's own layout — walk -`tests/` for `.json` files and assume nothing about the arrangement, which is -what upstream's README asks a runner to do. The CAR files are read from the +So `scripts/fetch-mst-vectors.sh` clones the suite, and the runner reads it in +upstream's own layout — walk `tests/` for `.json` files and assume nothing +about the arrangement, which is what upstream's README asks a runner to do. + +It is cached **per machine**, at `$XDG_CACHE_HOME/vibescrobble/mst-test-suite`, +beside the configuration and state this project already keeps under the same +directories. Per checkout was the obvious place and the wrong one: the corpus +is immutable upstream data that varies with nothing, this project is worked +from several worktrees at once, and each of them wanted its own 72MB and failed +four tests until somebody fetched one. A clone left in a checkout's own +`.cache/` is moved rather than re-downloaded on the first run. `MST_TEST_SUITE` +points at a corpus of your own and wins over both. + +One clone shared by every checkout means a plain run that fast-forwarded it +would move the corpus under whatever else is reading it, so a corpus already +there is left alone and reported. `--update`, or a ref, changes it on purpose. The CAR files are read from the same clone: 128 of them, 63kB, one tree apiece and no record blocks in any of them, because a diff never looks inside a record. diff --git a/plan/local-dev.md b/plan/local-dev.md index f22b9c1c..3db9d9cc 100644 --- a/plan/local-dev.md +++ b/plan/local-dev.md @@ -41,6 +41,19 @@ that is what fills a machine: see the dev profile under Done. ## Done +- [x] **The Merkle search tree corpus is cached per machine, not per checkout.** + It is 72MB of immutable upstream test data that varies with nothing, and + it used to live in each checkout's own `.cache/`: this machine carried + two copies within a day of the suite landing, and every new worktree + failed four tests until somebody fetched one. It now sits at + `$XDG_CACHE_HOME/vibescrobble/mst-test-suite`, beside the configuration + and state this project already keeps under the same directories, and a + clone left in a checkout is moved rather than re-downloaded. + + One clone shared by every checkout has a cost worth naming: a plain run + that fast-forwarded it would move the corpus under whatever else is + reading it. So a corpus already there is left alone and reported, and + `--update` is what changes it. - [x] **A test that failed about once in twelve suite runs, found and fixed.** `a_proof_missing_a_node_does_not_verify` asserted that a record's path through the Merkle search tree was more than one node deep. A record key diff --git a/scripts/fetch-mst-vectors.sh b/scripts/fetch-mst-vectors.sh index 361ace00..edad3cdf 100755 --- a/scripts/fetch-mst-vectors.sh +++ b/scripts/fetch-mst-vectors.sh @@ -4,22 +4,70 @@ # Upstream is , MIT. It is # 16,384 test cases in 16,384 JSON files and 24MB, and unlike the interop # vectors beside it — 96kB, and vendored under vendor/ — it is far too large -# to carry in this repository. So it is cloned here, into a directory git -# ignores, and the conformance runner fails with a pointer to this script when -# it is not there. +# to carry in this repository. So it is cloned, and the conformance runner +# fails with a pointer to this script when it is not there. # -# Re-running updates an existing clone rather than re-cloning it. A ref can be -# passed as the first argument to pin an older one. +# # It is cached per machine, not per checkout +# +# The corpus is immutable upstream test data. Nothing about it varies with the +# branch you are on, and this project is worked from several worktrees at once +# — each one used to want its own 72MB copy, and each new one failed four tests +# until somebody ran this. So it lives under `$XDG_CACHE_HOME/vibescrobble/`, +# beside the configuration in `$XDG_CONFIG_HOME/vibescrobble/` and the state in +# `$XDG_STATE_HOME/vibescrobble/` that the rest of this project already keeps +# there. +# +# A clone left in a checkout's own `.cache/` from before that is moved here on +# the first run rather than re-downloaded. +# +# # Updating is deliberate +# +# One clone is shared by every checkout on the machine, so a plain run that +# fast-forwarded it would move the corpus under whatever else is reading it. A +# corpus that is already there is therefore left alone and reported. Pass +# `--update`, or a ref, to change it on purpose. set -euo pipefail UPSTREAM="https://github.com/DavidBuchanan314/mst-test-suite.git" cd "$(dirname "$0")/.." -dest=".cache/mst-test-suite" -ref="${1:-HEAD}" +# Where the runner looks. Kept in step with `suite_dir` in +# `crates/vibescrobble/tests/conformance/mst_vectors.rs`, and checked against it +# by `the_fetch_script_and_the_runner_agree_on_where_the_corpus_is`. +cache_home="${XDG_CACHE_HOME:-$HOME/.cache}" +dest="$cache_home/vibescrobble/mst-test-suite" + +# What a checkout used to keep, and may still. +legacy=".cache/mst-test-suite" + +update=0 +ref="" +for arg in "$@"; do + case "$arg" in + --update) update=1 ;; + -*) echo "unknown option: $arg" >&2; exit 2 ;; + *) ref="$arg"; update=1 ;; + esac +done + +if [ -d "$legacy/.git" ] && [ ! -d "$dest/.git" ]; then + echo "moving $legacy to $dest" + mkdir -p "$(dirname "$dest")" + mv "$legacy" "$dest" +fi + +if [ -d "$dest/.git" ] && [ "$update" -eq 0 ]; then + commit="$(git -C "$dest" rev-parse HEAD)" + cases="$(find "$dest/tests" -name '*.json' 2>/dev/null | wc -l)" + echo "$dest is at $commit: $cases cases" + echo " already there; --update to move it, which every checkout on this machine shares" + exit 0 +fi + +ref="${ref:-HEAD}" if [ -d "$dest/.git" ]; then - echo "updating $dest" + echo "updating $dest to $ref" git -C "$dest" fetch --quiet --depth 1 origin "$ref" else echo "cloning $UPSTREAM into $dest"