From d3db2a570d757affe4406e7f9624b8a5e25180a9 Mon Sep 17 00:00:00 2001 From: Jer Miller Date: Mon, 27 Jul 2026 15:54:14 -0600 Subject: [PATCH] chore(backup): revert the sandbox-profile restic runner hooks These hooks existed only to serve the sandbox_profile harness, which was deleted in c8563df9f; they had zero product callers. Revert the backup side to 8e2ec2632. This removes run_restic_json_records and ResticJsonRecordsResult, and restores run_restic to its pre-arc single-subprocess signature and ResticResult to its 5-field shape. It also deletes solstone/think/json_codec.py, whose sole importer was backup/runner.py. state.get_daily_key, readiness.inspect_restic_ready, s3_wipe.list_prefix_contents, and rclone_install.check_rclone_ready go with the removed harness surface. readiness.check_restic_ready keeps its real product caller at backup/install.py:216, which passes no version_timeout, so that call site needed no edit. The five backup test files were sandbox-arc-only since baseline, so reverting them restores the pre-existing assertions byte-identical. The design doc no longer documents the removed runner API. Co-Authored-By: Claude Opus 5 (1M context) --- docs/design/spb-l2-keys-destination.md | 8 +- solstone/think/backup/rclone_install.py | 20 +- solstone/think/backup/readiness.py | 85 +-- solstone/think/backup/runner.py | 364 +------------ solstone/think/backup/s3_wipe.py | 50 +- solstone/think/backup/state.py | 11 - solstone/think/json_codec.py | 19 - tests/test_backup_acquire.py | 90 ---- tests/test_backup_rclone_acquire.py | 44 -- tests/test_backup_runner.py | 679 +----------------------- tests/test_backup_s3_wipe.py | 76 --- tests/test_backup_state.py | 38 -- 12 files changed, 31 insertions(+), 1453 deletions(-) delete mode 100644 solstone/think/json_codec.py diff --git a/docs/design/spb-l2-keys-destination.md b/docs/design/spb-l2-keys-destination.md index 07fa6a4af..2a50f73b7 100644 --- a/docs/design/spb-l2-keys-destination.md +++ b/docs/design/spb-l2-keys-destination.md @@ -153,10 +153,7 @@ repository string. ## Restic invocation model -`run_restic` remains the ordinary restic invocation path in source code. -SPB proof runs use `run_restic_json_records` for `backup --stdin`, `ls --long`, and `restore`; `init` remains human mode on `run_restic`. -Their JSON proof contract is closed over record kinds/cardinality but open to extra fields inside accepted records. -SPB proof runs also pass `--no-cache`, avoiding a restic cache under the operator's home during the proof. +`run_restic` remains the only restic invocation path in source code. Runner extension: @@ -306,9 +303,6 @@ Logging, if any, is limited to returncode and reason code. - `run_restic(..., pass_fds: tuple[int, ...] = ()) -> ResticResult` Existing behavior with optional FD inheritance for pipe-backed password files. -- `run_restic_json_records(...) -> ResticJsonRecordsResult` - Strict-UTF-8 JSON Lines from raw child stdout before diagnostic redaction, - zero-exit/cleanup-verified only, returned through one-shot opaque records. Read/write naming: diff --git a/solstone/think/backup/rclone_install.py b/solstone/think/backup/rclone_install.py index 08f71f91e..2c246e406 100644 --- a/solstone/think/backup/rclone_install.py +++ b/solstone/think/backup/rclone_install.py @@ -147,19 +147,13 @@ def _sentinel_payload( } -def check_rclone_ready( - tool_dir: Path | None = None, - *, - version_timeout: float = 10.0, -) -> Path | None: - os_name, arch = _platform_info() - resolved_tool_dir = tool_dir if tool_dir is not None else _tool_dir(os_name) - sentinel_path = _sentinel_path(resolved_tool_dir) +def _check_ready(tool_dir: Path, os_name: str, arch: str) -> Path | None: + sentinel_path = _sentinel_path(tool_dir) try: payload = json.loads(sentinel_path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return None - binary_path = _binary_path(resolved_tool_dir) + binary_path = _binary_path(tool_dir) expected = _sentinel_payload( os_name, arch, @@ -182,7 +176,7 @@ def check_rclone_ready( check=False, capture_output=True, text=True, - timeout=version_timeout, + timeout=10, ) except (OSError, subprocess.TimeoutExpired): return None @@ -201,10 +195,7 @@ def ensure_rclone( os_name, arch = _platform_info() resolved_tool_dir = tool_dir if tool_dir is not None else _tool_dir(os_name) if not force: - ready_path = check_rclone_ready( - resolved_tool_dir, - version_timeout=10.0, - ) + ready_path = _check_ready(resolved_tool_dir, os_name, arch) if ready_path is not None: return ready_path @@ -244,7 +235,6 @@ __all__ = [ "RCLONE_BUNDLE_ENV", "RCLONE_VERSION", "RCLONE_ZIP_SHA256", - "check_rclone_ready", "ensure_rclone", "select_rclone_asset", ] diff --git a/solstone/think/backup/readiness.py b/solstone/think/backup/readiness.py index bf5c86c60..e3c7ada27 100644 --- a/solstone/think/backup/readiness.py +++ b/solstone/think/backup/readiness.py @@ -14,7 +14,6 @@ import hashlib import json import os import platform -import signal import subprocess import sys from pathlib import Path @@ -147,13 +146,7 @@ def _sentinel_ready( return sha256 if isinstance(sha256, str) and sha256 else None -def _restic_version_ok( - binary_path: Path, - *, - timeout: float | None = None, -) -> bool: - if timeout is not None: - return _restic_version_ok_bounded(binary_path, timeout=timeout) +def _restic_version_ok(binary_path: Path) -> bool: try: result = subprocess.run( [str(binary_path), "version"], @@ -166,40 +159,6 @@ def _restic_version_ok( return result.returncode == 0 and f"restic {RESTIC_VERSION}" in result.stdout -def _restic_version_ok_bounded(binary_path: Path, *, timeout: float) -> bool: - try: - proc = subprocess.Popen( - [str(binary_path), "version"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - close_fds=True, - start_new_session=True, - ) - except OSError: - return False - try: - stdout, _stderr = proc.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - try: - os.killpg(proc.pid, signal.SIGTERM) - except OSError: - pass - try: - proc.wait(timeout=0.5) - except subprocess.TimeoutExpired: - try: - os.killpg(proc.pid, signal.SIGKILL) - except OSError: - pass - try: - proc.wait(timeout=0.5) - except subprocess.TimeoutExpired: - pass - return False - return proc.returncode == 0 and f"restic {RESTIC_VERSION}" in stdout - - def _verify_binary(binary_path: Path, expected_sha256: str) -> bool: if not binary_path.is_file() or not os.access(binary_path, os.X_OK): return False @@ -210,11 +169,7 @@ def _verify_binary(binary_path: Path, expected_sha256: str) -> bool: return actual_sha256 == expected_sha256 -def check_restic_ready( - tool_dir: Path | None = None, - *, - version_timeout: float | None = None, -) -> Path | None: +def check_restic_ready(tool_dir: Path | None = None) -> Path | None: os_name, arch = _platform_info() resolved_tool_dir = tool_dir if tool_dir is not None else _tool_dir(os_name) binary_path = _binary_path(resolved_tool_dir) @@ -226,42 +181,8 @@ def check_restic_ready( ) if expected_sha256 is None: return None - if version_timeout is None: - if not _restic_version_ok(binary_path): - return None - elif not _restic_version_ok(binary_path, timeout=version_timeout): + if not _restic_version_ok(binary_path): return None if not _verify_binary(binary_path, expected_sha256): return None return binary_path - - -def inspect_restic_ready( - tool_dir: Path | None = None, - *, - version_timeout: float = 5.0, -) -> tuple[Path | None, str | None]: - ready_path = check_restic_ready(tool_dir, version_timeout=version_timeout) - if ready_path is not None: - return ready_path, None - - os_name, arch = _platform_info() - resolved_tool_dir = tool_dir if tool_dir is not None else _tool_dir(os_name) - binary_path = _binary_path(resolved_tool_dir) - binary_present = binary_path.is_file() and os.access(binary_path, os.X_OK) - if not binary_present: - return None, "restic_missing" - - expected_sha256 = _sentinel_ready( - _load_sentinel(_sentinel_path(resolved_tool_dir)), - os_name, - arch, - binary_path, - ) - if expected_sha256 is None: - return None, "restic_incompatible" - if not _restic_version_ok_bounded(binary_path, timeout=version_timeout): - return None, "restic_incompatible" - if not _verify_binary(binary_path, expected_sha256): - return None, "restic_incompatible" - return None, "restic_incompatible" diff --git a/solstone/think/backup/runner.py b/solstone/think/backup/runner.py index a5e6ed8a1..ee456f358 100644 --- a/solstone/think/backup/runner.py +++ b/solstone/think/backup/runner.py @@ -7,19 +7,12 @@ from __future__ import annotations import json as json_module import os -import signal import subprocess from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass from pathlib import Path from typing import Any -from solstone.think import json_codec - -_PROCESS_GROUP_CLEANUP_UNVERIFIED = "process_group_cleanup_unverified" -_RESTIC_JSON_STDOUT_REDACTED = "[redacted restic json stdout]" -_RESTIC_JSON_STDERR_REDACTED = "[redacted restic stderr]" - @dataclass(frozen=True) class ResticResult: @@ -30,79 +23,6 @@ class ResticResult: argv: tuple[str, ...] -@dataclass(frozen=True) -class _RawResticProcessResult: - returncode: int - stdout: bytes | None - stderr: bytes | None - cleanup_verified: bool - timed_out: bool - - -class ResticJsonRecordsResult: - __slots__ = ("_argv", "_records", "_returncode", "_stderr", "_stdout") - - def __init__( - self, - *, - returncode: int, - stdout: str, - stderr: str, - argv: tuple[str, ...], - records: tuple[object, ...] | None, - ) -> None: - self._returncode = returncode - self._stdout = stdout - self._stderr = stderr - self._argv = argv - self._records = records - - @property - def returncode(self) -> int: - return self._returncode - - @property - def stdout(self) -> str: - return self._stdout - - @property - def stderr(self) -> str: - return self._stderr - - @property - def argv(self) -> tuple[str, ...]: - return self._argv - - @property - def has_records(self) -> bool: - return self._records is not None - - def consume_records(self) -> tuple[object, ...]: - records = self._records - if records is None: - raise TypeError("restic JSON records are unavailable") - self._records = None - return records - - def __repr__(self) -> str: - return "ResticJsonRecordsResult()" - - def __eq__(self, other: object) -> bool: - return self is other - - def __hash__(self) -> int: - return id(self) - - def __reduce__(self) -> object: - raise TypeError("ResticJsonRecordsResult is not serializable") - - def __copy__(self) -> object: - raise TypeError("ResticJsonRecordsResult is not copyable") - - def __deepcopy__(self, _memo: dict[int, object]) -> object: - raise TypeError("ResticJsonRecordsResult is not copyable") - - def _scrub(text: str, secrets: Iterable[str | None]) -> str: scrubbed = text for secret in secrets: @@ -178,44 +98,6 @@ def _parse_json(text: str) -> Any | None: return parsed -def _reject_json_constant(_value: str) -> None: - raise ValueError("non-standard JSON constant") - - -def _parse_json_records(raw_stdout: bytes | None) -> tuple[object, ...] | None: - if not raw_stdout: - return None - try: - text = raw_stdout.decode("utf-8") - # ASCII LF is the only record separator; zero or one final LF is tolerated. - lines = text.split("\n") - if lines and lines[-1] == "": - lines.pop() - if not lines: - return None - records: list[object] = [] - for line in lines: - if not line.strip(): - return None - records.append( - json_module.loads( - line, - object_pairs_hook=json_codec.reject_duplicate_keys, - parse_constant=_reject_json_constant, - ) - ) - if not records: - return None - return tuple(records) - except ( - UnicodeDecodeError, - json_module.JSONDecodeError, - ValueError, - RecursionError, - ): - return None - - def _timeout_text(value: str | bytes | None) -> str: if value is None: return "" @@ -224,115 +106,6 @@ def _timeout_text(value: str | bytes | None) -> str: return value -def _decode_output(value: str | bytes | None) -> str: - if value is None: - return "" - if isinstance(value, bytes): - return value.decode(errors="replace") - return value - - -def _wait_timeout(proc: subprocess.Popen[bytes], timeout: float) -> bool: - try: - proc.wait(timeout=timeout) - except subprocess.TimeoutExpired: - return False - return True - - -def _terminate_process_group( - proc: subprocess.Popen[bytes], - *, - terminate_grace_s: float, - kill_grace_s: float, -) -> bool: - if proc.poll() is not None: - return True - try: - os.killpg(proc.pid, signal.SIGTERM) - except ProcessLookupError: - return True - except OSError: - return False - if _wait_timeout(proc, terminate_grace_s): - return _process_group_absent(proc.pid) - - try: - os.killpg(proc.pid, signal.SIGKILL) - except ProcessLookupError: - return True - except OSError: - return False - if not _wait_timeout(proc, kill_grace_s): - return False - return _process_group_absent(proc.pid) - - -def _process_group_absent(pgid: int) -> bool: - try: - os.killpg(pgid, 0) - except ProcessLookupError: - return True - except OSError: - return False - return False - - -def _run_restic_popen( - argv: Sequence[str], - *, - env: Mapping[str, str], - timeout: float | None, - pass_fds: tuple[int, ...], - process_group: bool, - stdin_bytes: bytes | None, - terminate_grace_s: float, - kill_grace_s: float, -) -> _RawResticProcessResult: - proc = subprocess.Popen( - argv, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - close_fds=True, - pass_fds=pass_fds, - start_new_session=process_group, - ) - try: - raw_stdout, raw_stderr = proc.communicate(input=stdin_bytes, timeout=timeout) - except subprocess.TimeoutExpired as exc: - cleanup_verified = True - if process_group: - cleanup_verified = _terminate_process_group( - proc, - terminate_grace_s=terminate_grace_s, - kill_grace_s=kill_grace_s, - ) - else: - proc.kill() - cleanup_verified = _wait_timeout(proc, kill_grace_s) - try: - raw_stdout, raw_stderr = proc.communicate(timeout=0) - except subprocess.TimeoutExpired: - raw_stdout = exc.output - raw_stderr = exc.stderr - return _RawResticProcessResult( - returncode=124, - stdout=raw_stdout, - stderr=raw_stderr, - cleanup_verified=cleanup_verified, - timed_out=True, - ) - return _RawResticProcessResult( - returncode=proc.returncode, - stdout=raw_stdout, - stderr=raw_stderr, - cleanup_verified=True, - timed_out=False, - ) - - def reason_for_returncode(returncode: int) -> str: return { 3: "incomplete", @@ -365,75 +138,41 @@ def run_restic( max_repack_size: str | None = None, timeout: float | None = None, pass_fds: tuple[int, ...] = (), - process_group: bool = False, - stdin_bytes: bytes | None = None, - scrub_values: Iterable[str | None] = (), - terminate_grace_s: float = 3.0, - kill_grace_s: float = 5.0, ) -> ResticResult: env, secrets = _child_env(repository, password, backend_env) - scrub_secrets = (*secrets, *tuple(scrub_values)) argv = _build_argv(restic_path, args, json, max_repack_size) _guard_argv(argv, secrets) - safe_argv = tuple(_scrub(token, scrub_secrets) for token in argv) + safe_argv = tuple(argv) # Long-running/streaming backup mode is deferred: ManagedProcess.spawn # writes raw child stdout/stderr to health logs and the callosum logs tract, # and restic output may include presigned backend URLs or repo strings. - if not process_group and stdin_bytes is None: - try: - result = subprocess.run( - argv, - check=False, - capture_output=True, - text=True, - env=env, - timeout=timeout, - pass_fds=pass_fds, - ) - except subprocess.TimeoutExpired as exc: - stdout = _scrub(_timeout_text(exc.stdout), scrub_secrets) - stderr = _scrub(_timeout_text(exc.stderr), scrub_secrets) - return ResticResult( - returncode=124, - stdout=stdout, - stderr=stderr, - json=None, - argv=safe_argv, - ) - - stdout = _scrub(result.stdout or "", scrub_secrets) - stderr = _scrub(result.stderr or "", scrub_secrets) - parsed_json = _parse_json(stdout) if json else None + try: + result = subprocess.run( + argv, + check=False, + capture_output=True, + text=True, + env=env, + timeout=timeout, + pass_fds=pass_fds, + ) + except subprocess.TimeoutExpired as exc: + stdout = _scrub(_timeout_text(exc.stdout), secrets) + stderr = _scrub(_timeout_text(exc.stderr), secrets) return ResticResult( - returncode=result.returncode, + returncode=124, stdout=stdout, stderr=stderr, - json=parsed_json, + json=None, argv=safe_argv, ) - raw_result = _run_restic_popen( - argv, - env=env, - timeout=timeout, - pass_fds=pass_fds, - process_group=process_group, - stdin_bytes=stdin_bytes, - terminate_grace_s=terminate_grace_s, - kill_grace_s=kill_grace_s, - ) - stderr_text = _decode_output(raw_result.stderr) - if not raw_result.cleanup_verified: - stderr_text = f"{stderr_text}\n{_PROCESS_GROUP_CLEANUP_UNVERIFIED}" - stdout = _scrub(_decode_output(raw_result.stdout), scrub_secrets) - stderr = _scrub(stderr_text, scrub_secrets) - if raw_result.timed_out: - parsed_json = None - else: - parsed_json = _parse_json(stdout) if json else None + stdout = _scrub(result.stdout or "", secrets) + stderr = _scrub(result.stderr or "", secrets) + parsed_json = _parse_json(stdout) if json else None return ResticResult( - returncode=raw_result.returncode, + returncode=result.returncode, stdout=stdout, stderr=stderr, json=parsed_json, @@ -441,72 +180,9 @@ def run_restic( ) -def run_restic_json_records( - args: Sequence[str], - *, - repository: str, - password: str, - restic_path: Path, - backend_env: Mapping[str, str | None] | None = None, - timeout: float | None = None, - stdin_bytes: bytes | None = None, - scrub_values: Iterable[str | None] = (), - terminate_grace_s: float = 3.0, - kill_grace_s: float = 5.0, -) -> ResticJsonRecordsResult: - env, secrets = _child_env(repository, password, backend_env) - scrub_secrets = (*secrets, *tuple(scrub_values)) - argv = _build_argv(restic_path, args, json=True, max_repack_size=None) - _guard_argv(argv, secrets) - safe_argv = tuple(_scrub(token, scrub_secrets) for token in argv) - - raw_result = _run_restic_popen( - argv, - env=env, - timeout=timeout, - pass_fds=(), - process_group=True, - stdin_bytes=stdin_bytes, - terminate_grace_s=terminate_grace_s, - kill_grace_s=kill_grace_s, - ) - returncode = raw_result.returncode - cleanup_verified = raw_result.cleanup_verified - raw_stdout = raw_result.stdout - raw_stderr = raw_result.stderr - del raw_result - - stdout_present = bool(raw_stdout) - stderr_present = bool(raw_stderr) - records = ( - _parse_json_records(raw_stdout) - if returncode == 0 and cleanup_verified - else None - ) - raw_stdout = None - raw_stderr = None - - stdout = _RESTIC_JSON_STDOUT_REDACTED if stdout_present else "" - if not cleanup_verified: - stderr = f"{_RESTIC_JSON_STDERR_REDACTED}\n{_PROCESS_GROUP_CLEANUP_UNVERIFIED}" - elif stderr_present: - stderr = _RESTIC_JSON_STDERR_REDACTED - else: - stderr = "" - return ResticJsonRecordsResult( - returncode=returncode, - stdout=stdout, - stderr=stderr, - argv=safe_argv, - records=records, - ) - - __all__ = [ "ResticResult", - "ResticJsonRecordsResult", "reason_for_returncode", "run_restic", - "run_restic_json_records", "select_summary", ] diff --git a/solstone/think/backup/s3_wipe.py b/solstone/think/backup/s3_wipe.py index cf65a67b4..262f00220 100644 --- a/solstone/think/backup/s3_wipe.py +++ b/solstone/think/backup/s3_wipe.py @@ -11,7 +11,6 @@ import hmac import logging import socket import ssl -import time import urllib.error import urllib.parse import urllib.request @@ -204,8 +203,6 @@ class _S3Client: region: str, opener: Callable[..., object], timeout: float, - deadline_monotonic: float | None = None, - monotonic: Callable[[], float] = time.monotonic, ) -> None: parsed = urllib.parse.urlparse(endpoint.rstrip("/")) if parsed.scheme not in {"http", "https"} or not parsed.netloc: @@ -218,16 +215,6 @@ class _S3Client: self.region = region self.opener = opener self.timeout = timeout - self.deadline_monotonic = deadline_monotonic - self.monotonic = monotonic - - def _request_timeout(self) -> float: - if self.deadline_monotonic is None: - return self.timeout - remaining = self.deadline_monotonic - self.monotonic() - if remaining <= 0: - raise _WipeFailure("timeout") - return min(self.timeout, remaining) def request( self, @@ -279,7 +266,7 @@ class _S3Client: method=method, ) try: - with self.opener(request, timeout=self._request_timeout()) as response: + with self.opener(request, timeout=self.timeout) as response: status = int(getattr(response, "status", response.getcode())) raw_body = response.read() except urllib.error.HTTPError as exc: @@ -373,40 +360,6 @@ class _S3Client: ) -def list_prefix_contents( - *, - endpoint: str, - bucket: str, - prefix: str, - access_key_id: str, - secret_access_key: str, - session_token: str, - region: str = "auto", - opener: Callable[..., object] = urllib.request.urlopen, - timeout: float = S3_WIPE_TIMEOUT_SECONDS, - budget_s: float | None = None, -) -> tuple[tuple[str, ...], tuple[tuple[str, str], ...]]: - deadline_monotonic = None - if budget_s is not None: - deadline_monotonic = time.monotonic() + budget_s - client = _S3Client( - endpoint=endpoint, - access_key_id=access_key_id, - secret_access_key=secret_access_key, - session_token=session_token, - region=region, - opener=opener, - timeout=timeout, - deadline_monotonic=deadline_monotonic, - ) - keys = tuple(client.list_objects(bucket=bucket, prefix=prefix)) - uploads = tuple( - (upload.key, upload.upload_id) - for upload in client.list_uploads(bucket=bucket, prefix=prefix) - ) - return keys, uploads - - def wipe_prefix( *, endpoint: str, @@ -450,6 +403,5 @@ def wipe_prefix( __all__ = [ "S3_WIPE_TIMEOUT_SECONDS", "WipeResult", - "list_prefix_contents", "wipe_prefix", ] diff --git a/solstone/think/backup/state.py b/solstone/think/backup/state.py index eeb15adee..b9bef7ebb 100644 --- a/solstone/think/backup/state.py +++ b/solstone/think/backup/state.py @@ -7,7 +7,6 @@ from __future__ import annotations import copy from dataclasses import dataclass -from pathlib import Path from typing import Any from solstone.think.backup.destination import Destination @@ -147,15 +146,6 @@ def get_backup_config() -> dict[str, Any]: return merge_backup_config(config) -def get_daily_key(journal_path: str | Path | None = None) -> str | None: - if journal_path is None: - config = get_backup_config() - else: - config = merge_backup_config(read_journal_config(journal_path)) - daily_key = config["daily_key"] - return daily_key if isinstance(daily_key, str) and daily_key else None - - def get_destination() -> Destination | None: destination = get_backup_config()["destination"] repository = destination.get("repository") @@ -542,7 +532,6 @@ __all__ = [ "clear_backup_config", "generate_and_store_keys", "get_backup_config", - "get_daily_key", "get_destination", "get_keys", "merge_backup_config", diff --git a/solstone/think/json_codec.py b/solstone/think/json_codec.py deleted file mode 100644 index 757036019..000000000 --- a/solstone/think/json_codec.py +++ /dev/null @@ -1,19 +0,0 @@ -# SPDX-License-Identifier: AGPL-3.0-only -# Copyright (c) 2026 sol pbc - -"""Shared JSON decoding helpers.""" - -from __future__ import annotations - -from typing import Any - - -def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - seen: set[str] = set() - result: dict[str, Any] = {} - for key, value in pairs: - if key in seen: - raise ValueError("duplicate key") - seen.add(key) - result[key] = value - return result diff --git a/tests/test_backup_acquire.py b/tests/test_backup_acquire.py index 8be3975d0..ae0988ad5 100644 --- a/tests/test_backup_acquire.py +++ b/tests/test_backup_acquire.py @@ -110,96 +110,6 @@ def test_ensure_restic_reuses_ready_sentinel_without_downloading( assert calls == 0 -def test_inspect_restic_ready_distinguishes_missing( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - _patch_linux_amd64(monkeypatch) - - assert readiness.inspect_restic_ready(tmp_path) == (None, "restic_missing") - - -def test_inspect_restic_ready_distinguishes_incompatible_present_binary( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - _patch_linux_amd64(monkeypatch) - binary = tmp_path / "restic" - binary.write_text("#!/bin/sh\n", encoding="utf-8") - binary.chmod(0o755) - - assert readiness.inspect_restic_ready(tmp_path) == ( - None, - "restic_incompatible", - ) - - -def test_inspect_restic_ready_bounds_authoritative_version_probe( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - _patch_linux_amd64(monkeypatch) - binary = tmp_path / "restic" - payload = b"fake restic binary" - binary.write_bytes(payload) - binary.chmod(0o755) - (tmp_path / ".install-complete").write_text( - json.dumps( - { - "schema_version": readiness.RESTIC_SCHEMA_VERSION, - "tool": readiness.RESTIC_TOOL, - "version": readiness.RESTIC_VERSION, - "sha256": _sha256(payload), - "platform": {"os": "linux", "arch": "amd64"}, - "binary_path": str(binary), - } - ), - encoding="utf-8", - ) - procs: list[object] = [] - - class HangingProc: - pid = 12345 - returncode: int | None = None - alive = True - - def communicate(self, *, timeout: float): - assert timeout == 0.01 - raise readiness.subprocess.TimeoutExpired([str(binary), "version"], timeout) - - def wait(self, *, timeout: float) -> int: - assert timeout == 0.5 - if self.alive: - raise readiness.subprocess.TimeoutExpired( - [str(binary), "version"], timeout - ) - assert self.returncode is not None - return self.returncode - - def fake_popen(argv: list[str], **kwargs): - assert argv == [str(binary), "version"] - assert kwargs["start_new_session"] is True - proc = HangingProc() - procs.append(proc) - return proc - - def fake_killpg(pgid: int, sig: int) -> None: - assert pgid == 12345 - proc = procs[-1] - proc.alive = False - proc.returncode = -sig - - monkeypatch.setattr(readiness.subprocess, "Popen", fake_popen) - monkeypatch.setattr(readiness.os, "killpg", fake_killpg) - - assert readiness.inspect_restic_ready(tmp_path, version_timeout=0.01) == ( - None, - "restic_incompatible", - ) - assert len(procs) == 2 - assert all(not proc.alive for proc in procs) - - @pytest.mark.parametrize( ("os_name", "arch", "expected_filename"), [ diff --git a/tests/test_backup_rclone_acquire.py b/tests/test_backup_rclone_acquire.py index f9fbc4f76..513a36b52 100644 --- a/tests/test_backup_rclone_acquire.py +++ b/tests/test_backup_rclone_acquire.py @@ -100,50 +100,6 @@ def test_ensure_rclone_reuses_verified_install( assert calls == 1 -def test_check_rclone_ready_is_read_only_and_honors_timeout( - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, -) -> None: - binary = tmp_path / "rclone" - payload = _fake_binary() - binary.write_bytes(payload) - binary.chmod(0o755) - (tmp_path / ".install-complete").write_text( - json.dumps( - { - "schema_version": rclone_install.RCLONE_SCHEMA_VERSION, - "tool": "rclone", - "version": rclone_install.RCLONE_VERSION, - "sha256": _sha256(payload), - "platform": {"os": "linux", "arch": "amd64"}, - "binary_path": str(binary), - } - ), - encoding="utf-8", - ) - captured: dict[str, object] = {} - - def fake_run(argv: list[str], **kwargs): - captured["argv"] = argv - captured["timeout"] = kwargs["timeout"] - return rclone_install.subprocess.CompletedProcess( - argv, - 0, - stdout="rclone v1.74.4\n", - stderr="", - ) - - monkeypatch.setattr(rclone_install, "_platform_info", lambda: ("linux", "amd64")) - monkeypatch.setattr(rclone_install.subprocess, "run", fake_run) - - assert ( - rclone_install.check_rclone_ready(tmp_path, version_timeout=5.0) - == tmp_path / "rclone" - ) - assert captured["argv"] == [str(binary), "version"] - assert captured["timeout"] == 5.0 - - def test_ensure_rclone_reinstalls_when_ready_binary_cannot_be_hashed( monkeypatch: pytest.MonkeyPatch, tmp_path: Path, diff --git a/tests/test_backup_runner.py b/tests/test_backup_runner.py index 7d315ab27..b8dbe1e5b 100644 --- a/tests/test_backup_runner.py +++ b/tests/test_backup_runner.py @@ -3,13 +3,8 @@ from __future__ import annotations -import copy -import dataclasses -import inspect import json -import pickle import subprocess -import traceback from pathlib import Path from typing import Any @@ -17,13 +12,6 @@ import pytest from solstone.think.backup import runner -SNAPSHOT_ID = "5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d5a1d" -LOGICAL_SOURCE_PATH = "/spb/source.bin" - - -def _jsonl(*records: object) -> bytes: - return ("\n".join(json.dumps(record) for record in records) + "\n").encode() - def test_run_restic_builds_safe_argv_and_minimal_env( monkeypatch: pytest.MonkeyPatch, @@ -77,36 +65,6 @@ def test_run_restic_builds_safe_argv_and_minimal_env( assert captured["pass_fds"] == () -def test_run_restic_default_path_does_not_use_popen( - monkeypatch: pytest.MonkeyPatch, -) -> None: - calls: dict[str, Any] = {} - - def fake_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - calls["argv"] = argv - calls["kwargs"] = kwargs - return subprocess.CompletedProcess(argv, 0, stdout="", stderr="") - - def fail_popen(*_args: Any, **_kwargs: Any) -> None: - raise AssertionError("default path should not use Popen") - - monkeypatch.setattr(runner.subprocess, "run", fake_run) - monkeypatch.setattr(runner.subprocess, "Popen", fail_popen) - - result = runner.run_restic( - ["snapshots"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - ) - - assert result.returncode == 0 - assert calls["argv"] == ["/usr/bin/restic", "snapshots"] - assert calls["kwargs"]["text"] is True - assert calls["kwargs"]["capture_output"] is True - assert "input" not in calls["kwargs"] - - def test_run_restic_threads_pass_fds(monkeypatch: pytest.MonkeyPatch): captured: dict[str, Any] = {} @@ -136,501 +94,6 @@ def test_run_restic_threads_pass_fds(monkeypatch: pytest.MonkeyPatch): assert captured["pass_fds"] == (17,) -def test_run_restic_opt_in_process_group_threads_stdin_and_scrubs_argv( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: dict[str, Any] = {} - - class FakePopen: - returncode = 0 - pid = 12345 - - def __init__(self, argv: list[str], **kwargs: Any) -> None: - captured["argv"] = argv - captured["kwargs"] = kwargs - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - captured["input"] = input - captured["timeout"] = timeout - return ( - b'{"message":"repo-url snapshot-id"}\n', - b"stderr repo-url snapshot-id", - ) - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic( - ["ls", "snapshot-id"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - process_group=True, - stdin_bytes=b"payload", - timeout=9, - scrub_values=("repo-url", "snapshot-id"), - ) - - assert captured["kwargs"]["start_new_session"] is True - assert captured["kwargs"]["close_fds"] is True - assert captured["input"] == b"payload" - assert captured["timeout"] == 9 - assert "snapshot-id" not in result.argv - assert "snapshot-id" not in result.stdout - assert "repo-url" not in result.stderr - assert result.returncode == 0 - - -def test_run_restic_json_records_parses_raw_before_scrub( - monkeypatch: pytest.MonkeyPatch, -) -> None: - captured: dict[str, Any] = {} - raw_stdout = _jsonl( - {"message_type": "snapshot", "id": SNAPSHOT_ID, "paths": [LOGICAL_SOURCE_PATH]}, - {"message_type": "node", "path": LOGICAL_SOURCE_PATH, "type": "file"}, - ) - - class FakePopen: - returncode = 0 - pid = 12345 - - def __init__(self, argv: list[str], **kwargs: Any) -> None: - captured["argv"] = argv - captured["kwargs"] = kwargs - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - captured["input"] = input - captured["timeout"] = timeout - return raw_stdout, b"stderr repo-url " + SNAPSHOT_ID.encode() - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic_json_records( - ["ls", SNAPSHOT_ID], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - backend_env={"AWS_SECRET_ACCESS_KEY": "backend-secret"}, - timeout=9, - stdin_bytes=b"payload", - scrub_values=("repo-url", SNAPSHOT_ID, LOGICAL_SOURCE_PATH), - ) - - assert captured["kwargs"]["start_new_session"] is True - assert captured["kwargs"]["close_fds"] is True - assert captured["kwargs"]["pass_fds"] == () - assert captured["input"] == b"payload" - assert captured["timeout"] == 9 - assert captured["argv"][-1] == "--json" - assert result.stdout == runner._RESTIC_JSON_STDOUT_REDACTED - assert result.stderr == runner._RESTIC_JSON_STDERR_REDACTED - assert SNAPSHOT_ID not in " ".join(result.argv) - assert LOGICAL_SOURCE_PATH not in " ".join(result.argv) - assert result.has_records is True - records = result.consume_records() - assert records[0]["id"] == SNAPSHOT_ID - assert records[0]["paths"] == [LOGICAL_SOURCE_PATH] - assert records[1]["path"] == LOGICAL_SOURCE_PATH - assert result.has_records is False - with pytest.raises(TypeError, match="unavailable"): - result.consume_records() - - -@pytest.mark.parametrize( - ("raw_stdout", "expected_records"), - [ - (b'{"a":1}', ({"a": 1},)), - (b'{"a":1}\n', ({"a": 1},)), - (b'{"a":1}\n{"b":2}\n', ({"a": 1}, {"b": 2})), - (b'{"a":1}\r\n{"b":2}\r\n', ({"a": 1}, {"b": 2})), - ], - ids=[ - "no_final_lf", - "one_final_lf", - "lf_records", - "crlf_records", - ], -) -def test_run_restic_json_records_accepts_lf_record_separators( - monkeypatch: pytest.MonkeyPatch, - raw_stdout: bytes, - expected_records: tuple[object, ...], -) -> None: - class FakePopen: - returncode = 0 - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - pass - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - return raw_stdout, b"" - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic_json_records( - ["backup", "--stdin"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - ) - - assert result.has_records is True - assert result.consume_records() == expected_records - - -@pytest.mark.parametrize( - "raw_stdout", - [ - b'{"a":1}\x0b{"b":2}', - b'{"a":1}\x0c{"b":2}', - b'{"a":1}\x1c{"b":2}', - b'{"a":1}\x1d{"b":2}', - b'{"a":1}\x1e{"b":2}', - b'{"a":1}\xc2\x85{"b":2}', - b'{"a":1}\xe2\x80\xa8{"b":2}', - b'{"a":1}\xe2\x80\xa9{"b":2}', - b'\n{"a":1}', - b'{"a":1}\n\n{"b":2}', - b'{"a":1}\n\n', - b'{"a":1}\r\n\r\n', - ], - ids=[ - "vt_separator", - "ff_separator", - "fs_separator", - "gs_separator", - "rs_separator", - "nel_separator", - "ls_separator", - "ps_separator", - "leading_blank", - "middle_blank", - "double_terminal_lf", - "separator_only_trailing_crlf", - ], -) -def test_run_restic_json_records_rejects_non_lf_record_boundaries( - monkeypatch: pytest.MonkeyPatch, - raw_stdout: bytes, -) -> None: - class FakePopen: - returncode = 0 - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - pass - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - return raw_stdout, b"" - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic_json_records( - ["backup", "--stdin"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - ) - - assert result.has_records is False - - -@pytest.mark.parametrize( - "raw_stdout", - [ - b"", - b"\xff", - b'{"message_type":', - b'\n{"message_type":"summary"}\n', - b'{"message_type":"status"}\n\n{"message_type":"summary"}\n', - b'{"message_type":"summary"}\n\n', - b"NaN\n", - b"Infinity\n", - b"-Infinity\n", - b'{"message_type":"summary","message_type":"summary"}\n', - b'{"outer":{"middle":{"key":1,"key":2}}}\n', - ], - ids=[ - "empty", - "invalid_utf8", - "malformed", - "blank_leading_record", - "blank_middle_record", - "blank_trailing_record", - "nan", - "infinity", - "negative_infinity", - "duplicate_top_level", - "duplicate_nested_depth_two", - ], -) -def test_run_restic_json_records_rejections_are_content_free( - monkeypatch: pytest.MonkeyPatch, - raw_stdout: bytes, -) -> None: - canary = "spb/source.bin" - - class FakePopen: - returncode = 0 - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - pass - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - return raw_stdout, b"" - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic_json_records( - ["backup", "--stdin-filename", f"/{canary}"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - scrub_values=(f"/{canary}",), - ) - - assert result.has_records is False - assert result.stdout in {"", runner._RESTIC_JSON_STDOUT_REDACTED} - with pytest.raises(TypeError) as excinfo: - result.consume_records() - assert canary not in str(excinfo.value) - rendered = "".join( - traceback.format_exception( - type(excinfo.value), - excinfo.value, - excinfo.value.__traceback__, - ) - ) - assert canary not in rendered - - -def test_run_restic_json_records_preserves_all_json_value_types( - monkeypatch: pytest.MonkeyPatch, -) -> None: - raw_stdout = b'{"object":true}\n[1,2]\n"text"\n7\ntrue\nfalse\nnull\n' - - class FakePopen: - returncode = 0 - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - pass - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - return raw_stdout, b"" - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic_json_records( - ["backup", "--stdin"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - ) - - assert result.consume_records() == ( - {"object": True}, - [1, 2], - "text", - 7, - True, - False, - None, - ) - - -@pytest.mark.parametrize( - ("mode", "expected_returncode", "expected_stderr"), - [ - ("timeout", 124, runner._RESTIC_JSON_STDERR_REDACTED), - ("nonzero", 7, runner._RESTIC_JSON_STDERR_REDACTED), - ( - "cleanup_unverified", - 124, - ( - f"{runner._RESTIC_JSON_STDERR_REDACTED}\n" - f"{runner._PROCESS_GROUP_CLEANUP_UNVERIFIED}" - ), - ), - ], -) -def test_run_restic_json_records_parse_gate_precedes_parser( - monkeypatch: pytest.MonkeyPatch, - mode: str, - expected_returncode: int, - expected_stderr: str, -) -> None: - parse_calls = 0 - - def fake_parse(_raw_stdout: bytes | None) -> tuple[object, ...] | None: - nonlocal parse_calls - parse_calls += 1 - return ({"message_type": "summary"},) - - class FakePopen: - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - self.returncode = 7 if mode == "nonzero" else 0 - self._calls = 0 - - def poll(self) -> None: - return None - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - self._calls += 1 - if mode in {"timeout", "cleanup_unverified"} and self._calls == 1: - raise subprocess.TimeoutExpired( - ["restic"], - timeout=1, - output=b'{"message_type":"summary"}\n', - stderr=b"stderr", - ) - return b'{"message_type":"summary"}\n', b"stderr" - - monkeypatch.setattr(runner, "_parse_json_records", fake_parse) - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - monkeypatch.setattr( - runner, - "_terminate_process_group", - lambda *_args, **_kwargs: mode != "cleanup_unverified", - ) - - result = runner.run_restic_json_records( - ["backup", "--stdin"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - ) - - assert result.returncode == expected_returncode - assert result.stderr == expected_stderr - assert result.has_records is False - assert parse_calls == 0 - - -def test_restic_json_records_result_is_opaque_and_one_shot() -> None: - canaries = ("SECRET-CANARY", SNAPSHOT_ID, LOGICAL_SOURCE_PATH) - result = runner.ResticJsonRecordsResult( - returncode=0, - stdout=runner._RESTIC_JSON_STDOUT_REDACTED, - stderr=runner._RESTIC_JSON_STDERR_REDACTED, - argv=("restic", "[redacted]"), - records=({"message_type": "summary", "secret": canaries[0]},), - ) - same_shape = runner.ResticJsonRecordsResult( - returncode=0, - stdout=runner._RESTIC_JSON_STDOUT_REDACTED, - stderr=runner._RESTIC_JSON_STDERR_REDACTED, - argv=("restic", "[redacted]"), - records=({"message_type": "summary", "secret": canaries[0]},), - ) - - assert not hasattr(result, "__dict__") - assert repr(result) == "ResticJsonRecordsResult()" - assert result == result - assert result != same_shape - assert hash(result) == id(result) - for operation in (copy.copy, copy.deepcopy, pickle.dumps): - with pytest.raises(TypeError) as excinfo: - operation(result) - assert excinfo.value.__cause__ is None - rendered = "".join( - traceback.format_exception( - type(excinfo.value), - excinfo.value, - excinfo.value.__traceback__, - ) - ) - for canary in canaries: - assert canary not in str(excinfo.value) - assert canary not in rendered - - records = result.consume_records() - assert records == ({"message_type": "summary", "secret": canaries[0]},) - assert result.has_records is False - assert repr(result) == "ResticJsonRecordsResult()" - refusing_operations = ( - vars, - dataclasses.asdict, - json.dumps, - lambda value: value.consume_records(), - ) - for operation in refusing_operations: - with pytest.raises(TypeError) as excinfo: - operation(result) - assert excinfo.value.__cause__ is None - rendered = "".join( - traceback.format_exception( - type(excinfo.value), - excinfo.value, - excinfo.value.__traceback__, - ) - ) - for canary in canaries: - assert canary not in str(excinfo.value) - assert canary not in rendered - assert repr(result) == "ResticJsonRecordsResult()" - - -def test_run_restic_json_records_api_exposes_no_parser_or_raw_escape_hatch() -> None: - signature = inspect.signature(runner.run_restic_json_records) - assert tuple(signature.parameters) == ( - "args", - "repository", - "password", - "restic_path", - "backend_env", - "timeout", - "stdin_bytes", - "scrub_values", - "terminate_grace_s", - "kill_grace_s", - ) - assert not any( - parameter.kind is inspect.Parameter.VAR_KEYWORD - for parameter in signature.parameters.values() - ) - result = runner.ResticJsonRecordsResult( - returncode=0, - stdout="", - stderr="", - argv=("restic",), - records=None, - ) - for name in ("json", "raw_stdout", "raw_stderr", "raw_output", "records"): - assert not hasattr(result, name) - - def test_run_restic_scrubs_success_output_and_json( monkeypatch: pytest.MonkeyPatch, ): @@ -765,7 +228,7 @@ def test_run_restic_timeout_returns_scrubbed_result( raise subprocess.TimeoutExpired( argv, timeout=1, - output=b'{"message":"repo-password backend-secret SESS-TOKEN"}', + output=b"stdout repo-password backend-secret SESS-TOKEN", stderr=b"stderr repo-password backend-secret SESS-TOKEN", ) @@ -780,7 +243,6 @@ def test_run_restic_timeout_returns_scrubbed_result( "AWS_SECRET_ACCESS_KEY": "backend-secret", "AWS_SESSION_TOKEN": "SESS-TOKEN", }, - json=True, timeout=1, ) @@ -794,92 +256,6 @@ def test_run_restic_timeout_returns_scrubbed_result( assert result.json is None -@pytest.mark.parametrize("returncode", [0, 7, 124]) -def test_run_restic_popen_normal_completion_parses_json_for_returncode( - monkeypatch: pytest.MonkeyPatch, - returncode: int, -) -> None: - class FakePopen: - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - self.returncode = returncode - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - return b'{"message":"repo-password"}', b"stderr repo-password" - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - - result = runner.run_restic( - ["backup", "/tmp/data"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - json=True, - process_group=True, - ) - - assert result.returncode == returncode - assert "repo-password" not in result.stdout - assert "repo-password" not in result.stderr - assert result.json == {"message": "[redacted]"} - - -def test_run_restic_popen_timeout_json_is_none( - monkeypatch: pytest.MonkeyPatch, -) -> None: - class FakePopen: - pid = 12345 - - def __init__(self, _argv: list[str], **_kwargs: Any) -> None: - self.returncode = 0 - self._calls = 0 - - def poll(self) -> None: - return None - - def communicate( - self, - input: bytes | None = None, - timeout: float | None = None, - ) -> tuple[bytes, bytes]: - self._calls += 1 - if self._calls == 1: - raise subprocess.TimeoutExpired( - ["restic"], - timeout=1, - output=b'{"message":"repo-password"}', - stderr=b"stderr repo-password", - ) - return b'{"message":"repo-password"}', b"stderr repo-password" - - monkeypatch.setattr(runner.subprocess, "Popen", FakePopen) - monkeypatch.setattr( - runner, - "_terminate_process_group", - lambda *_args, **_kwargs: True, - ) - - result = runner.run_restic( - ["backup", "/tmp/data"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - json=True, - process_group=True, - timeout=1, - ) - - assert result.returncode == 124 - assert "repo-password" not in result.stdout - assert "repo-password" not in result.stderr - assert result.json is None - - def test_parse_json_lines_from_scrubbed_stdout( monkeypatch: pytest.MonkeyPatch, ): @@ -901,59 +277,6 @@ def test_parse_json_lines_from_scrubbed_stdout( assert result.json == [{"message": "[redacted]"}, {"message": "ok"}] -def test_run_restic_result_shape_and_dataclass_behavior_are_frozen( - monkeypatch: pytest.MonkeyPatch, -) -> None: - def fake_run(argv: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - return subprocess.CompletedProcess( - argv, - 3, - stdout='{"message":"repo-password backend-secret","status":"ok"}', - stderr="stderr repo-password backend-secret", - ) - - monkeypatch.setattr(runner.subprocess, "run", fake_run) - - result = runner.run_restic( - ["backup", "/tmp/data"], - repository="s3:safe-bucket/path", - password="repo-password", - restic_path=Path("/usr/bin/restic"), - backend_env={"AWS_SECRET_ACCESS_KEY": "backend-secret"}, - json=True, - ) - same = runner.ResticResult( - 3, - result.stdout, - result.stderr, - result.json, - result.argv, - ) - - assert tuple(field.name for field in dataclasses.fields(runner.ResticResult)) == ( - "returncode", - "stdout", - "stderr", - "json", - "argv", - ) - assert result.returncode == 3 - assert result.stdout == '{"message":"[redacted] [redacted]","status":"ok"}' - assert result.stderr == "stderr [redacted] [redacted]" - assert result.json == {"message": "[redacted] [redacted]", "status": "ok"} - assert result.argv == ("/usr/bin/restic", "backup", "/tmp/data", "--json") - assert result == same - assert repr(result).startswith("ResticResult(") - assert dataclasses.asdict(result) == { - "returncode": 3, - "stdout": result.stdout, - "stderr": result.stderr, - "json": result.json, - "argv": result.argv, - } - assert pickle.loads(pickle.dumps(result)) == result - - @pytest.mark.parametrize( ("returncode", "reason"), [ diff --git a/tests/test_backup_s3_wipe.py b/tests/test_backup_s3_wipe.py index fd5202c4e..c1dc4a8a7 100644 --- a/tests/test_backup_s3_wipe.py +++ b/tests/test_backup_s3_wipe.py @@ -12,8 +12,6 @@ import xml.etree.ElementTree as ET from dataclasses import dataclass from typing import Any -import pytest - from solstone.think.backup import s3_wipe ENDPOINT = "https://r2.example" @@ -31,7 +29,6 @@ class RequestRecord: url: str headers: dict[str, str] body: bytes - timeout: float class FakeResponse: @@ -65,7 +62,6 @@ class FakeOpener: url=request.full_url, headers={key.lower(): value for key, value in request.header_items()}, body=request.data or b"", - timeout=timeout, ) ) if not self.items: @@ -196,78 +192,6 @@ def test_populated_prefix_deletes_objects_then_aborts_uploads() -> None: assert "uploadId=upload-1" in opener.records[4].url -def test_wipe_prefix_lists_uploads_only_after_object_deletes() -> None: - opener = FakeOpener( - [ - FakeResponse(200, _list_objects([f"{PREFIX}object"])), - FakeResponse(200, _delete_result()), - FakeResponse(200, _list_uploads([(f"{PREFIX}multipart", "upload-1")])), - FakeResponse(204), - ] - ) - - result = _wipe(opener) - - assert result == s3_wipe.WipeResult("ok", None) - assert [record.method for record in opener.records] == [ - "GET", - "POST", - "GET", - "DELETE", - ] - assert "delete=" in opener.records[1].url - assert "uploads=" in opener.records[2].url - - -def test_list_prefix_contents_reads_objects_and_uploads_without_mutation() -> None: - opener = FakeOpener( - [ - FakeResponse( - 200, - _list_objects([f"{PREFIX}a"], truncated=True, token="next"), - ), - FakeResponse(200, _list_objects([f"{PREFIX}b"])), - FakeResponse( - 200, - _list_uploads([(f"{PREFIX}upload", "upload-1")]), - ), - ] - ) - - keys, uploads = s3_wipe.list_prefix_contents( - endpoint=ENDPOINT, - bucket=BUCKET, - prefix=PREFIX, - access_key_id=ACCESS_KEY, - secret_access_key=SECRET_KEY, - session_token=SESSION_TOKEN, - opener=opener, - timeout=7, - budget_s=30, - ) - - assert keys == (f"{PREFIX}a", f"{PREFIX}b") - assert uploads == ((f"{PREFIX}upload", "upload-1"),) - assert [record.method for record in opener.records] == ["GET", "GET", "GET"] - assert all(record.timeout <= 7 for record in opener.records) - - -def test_list_prefix_contents_propagates_ambiguous_pagination() -> None: - opener = FakeOpener([FakeResponse(200, _list_objects([], truncated=True))]) - - with pytest.raises(Exception, match="failed"): - s3_wipe.list_prefix_contents( - endpoint=ENDPOINT, - bucket=BUCKET, - prefix=PREFIX, - access_key_id=ACCESS_KEY, - secret_access_key=SECRET_KEY, - session_token=SESSION_TOKEN, - opener=opener, - timeout=1, - ) - - def test_list_objects_paginates_with_continuation_token() -> None: opener = FakeOpener( [ diff --git a/tests/test_backup_state.py b/tests/test_backup_state.py index 805c97320..1248da9b8 100644 --- a/tests/test_backup_state.py +++ b/tests/test_backup_state.py @@ -72,44 +72,6 @@ def test_partial_backup_section_gets_per_field_defaults( assert config["last_backup"] == state.BACKUP_DEFAULTS["last_backup"] -def test_get_daily_key_returns_only_daily_key( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setenv("SOLSTONE_JOURNAL", str(tmp_path)) - _write_config( - tmp_path, - {"backup": {"daily_key": "daily-secret", "recovery_key": "recovery-secret"}}, - ) - - assert state.get_daily_key() == "daily-secret" - - -def test_get_daily_key_can_read_path_explicit_journal( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - ambient = tmp_path / "ambient" - explicit = tmp_path / "explicit" - monkeypatch.setenv("SOLSTONE_JOURNAL", str(ambient)) - _write_config( - ambient, - {"backup": {"daily_key": "ambient-daily", "recovery_key": "ambient-recovery"}}, - ) - _write_config( - explicit, - { - "backup": { - "daily_key": "explicit-daily", - "recovery_key": "explicit-recovery", - } - }, - ) - - assert state.get_daily_key(explicit) == "explicit-daily" - assert state.get_daily_key() == "ambient-daily" - - def test_merge_backup_config_applies_defaults_to_raw_config() -> None: config = state.merge_backup_config( { -- 2.51.2