diff --git a/spindle/db/cache.go b/spindle/db/cache.go index 12afd3ab..67680258 100644 --- a/spindle/db/cache.go +++ b/spindle/db/cache.go @@ -326,13 +326,6 @@ func (d *DB) FindFallbackCacheEntry(ctx context.Context, repoDID, engine, key, e order by created_at desc limit 1`, repoDID, engine, key, excludeHash)) } -func (d *DB) FindFallbackCacheEntryForIdentity(ctx context.Context, repoDID, engine, prefix, excludeHash string) (*CacheEntry, error) { - return scanCacheEntry(d.QueryRowContext(ctx, ` - select `+cacheEntryColumns+` from cache_entries - where repo_did = ? and engine = ? and cache_key like ? || '%' - and cache_hash <> ? and cache_hash <> '' and state = 'ready' - order by created_at desc limit 1`, repoDID, engine, prefix, excludeHash)) -} func (d *DB) BeginCacheRestore(ctx context.Context, id string) (bool, error) { result, err := d.ExecContext(ctx, `update cache_entries set restore_count = restore_count + 1 where id = ? and state = 'ready'`, id) diff --git a/spindle/engine/cache_controller.go b/spindle/engine/cache_controller.go index a43b9169..9d92b2e3 100644 --- a/spindle/engine/cache_controller.go +++ b/spindle/engine/cache_controller.go @@ -66,65 +66,6 @@ func NewLocalCacheController(index *db.DB, store storage.Storage, repoDir string } } -// on a hash miss, the newest older generation still warms the build -// unusable entries degrade to a plain miss -func resolveCaches(ctx context.Context, l *slog.Logger, index *db.DB, repoDID, engine, repoPath, rev string, entries []models.CacheEntry) []models.CacheBinding { - resolved := make([]models.CacheBinding, 0, len(entries)) - for entryIndex, entry := range entries { - hash := "" - if len(entry.Hash) > 0 && repoPath != "" { - sum, missing := hashKeyFilesLegacy(ctx, repoPath, rev, entry.Hash) - for _, m := range missing { - l.Warn("cache hash file not in repo", "key", entry.Key, "path", m) - } - hash = sum - } - binding := models.CacheBinding{ - EntryIndex: entryIndex, - Paths: entry.Paths, - Key: entry.Key, - Hash: hash, - CompressionLevel: entry.CompressionLevel, - When: entry.When, - } - - found, err := index.FindCacheEntry(ctx, repoDID, engine, entry.Key, hash) - if errors.Is(err, sql.ErrNoRows) && hash != "" { - found, err = index.FindFallbackCacheEntry(ctx, repoDID, engine, entry.Key, hash) - if err == nil { - binding.RestoreName = entry.Key + "-" + found.CacheHash - } - } - if err != nil && !errors.Is(err, sql.ErrNoRows) { - l.Warn("cache lookup failed; entry will save but not restore", "key", entry.Key, "err", err) - } else if err == nil { - binding.RestoreID = found.ID - binding.RestoreKey = found.StorageKey - } - resolved = append(resolved, binding) - } - return resolved -} - -func hashKeyFilesLegacy(ctx context.Context, repoPath, rev string, paths []string) (string, []string) { - h := sha256.New() - var missing []string - hashed := 0 - for _, p := range paths { - blob, err := gitBlobId(ctx, repoPath, rev, p) - if err != nil { - missing = append(missing, p) - continue - } - fmt.Fprintf(h, "%s=%s\n", p, blob) - hashed++ - } - if hashed == 0 { - return "", missing - } - return hex.EncodeToString(h.Sum(nil))[:12], missing -} - func hashKeyFiles(ctx context.Context, repoPath, rev string, paths []string) (string, []string) { h := sha256.New() var missing []string @@ -227,12 +168,7 @@ func (c *localCacheController) Plan(ctx context.Context, pipeline *models.Pipeli cacheNamespace = models.CacheNamespace() } cacheEngine := workflow.Engine + "/" + cacheNamespace - var bindings []models.CacheBinding - if workflow.Name == "" && pipeline.TriggerMetadata == nil { - bindings = resolveCaches(ctx, c.logger, c.index, pipeline.RepoDid.String(), cacheEngine, repoPath, rev, workflow.Caches) - } else { - bindings = resolveCachesIdentity(ctx, c.logger, c.index, pipeline.RepoDid.String(), cacheEngine, repoPath, rev, workflow.Name, pipelineRef(pipeline.TriggerMetadata), workflow.Caches) - } + bindings := resolveCachesIdentity(ctx, c.logger, c.index, pipeline.RepoDid.String(), cacheEngine, repoPath, rev, workflow.Name, pipelineRef(pipeline.TriggerMetadata), workflow.Caches) now := time.Now() pendingUntil := now // pending rows stay live through the workflow deadline @@ -241,10 +177,7 @@ func (c *localCacheController) Plan(ctx context.Context, pipeline *models.Pipeli } for i := range bindings { id := uuid.NewString() - cacheKey := bindings[i].Key - if workflow.Name != "" || pipeline.TriggerMetadata != nil { - cacheKey = cacheIdentityKey(workflow.Name, pipelineRef(pipeline.TriggerMetadata), cacheKey) - } + cacheKey := cacheIdentityKey(workflow.Name, pipelineRef(pipeline.TriggerMetadata), bindings[i].Key) saveKey, err := models.CacheObjectKey(pipeline.RepoDid.String(), id) if err != nil { return nil, err diff --git a/spindle/engine/cache_identity_test.go b/spindle/engine/cache_identity_test.go index cb1110ac..6e021a6e 100644 --- a/spindle/engine/cache_identity_test.go +++ b/spindle/engine/cache_identity_test.go @@ -231,3 +231,27 @@ func TestCacheIdentityNoHashSharedAcrossCommits(t *testing.T) { t.Fatalf("restore = %q, want %q", got.RestoreKey, a.SaveKey) } } + +func TestCacheIdentityMissingHashFiles(t *testing.T) { + d := newCacheTestDB(t) + repoPath := filepath.Join(t.TempDir(), "repo") + rev := identityTestCommit(t, repoPath, map[string]string{"go.mod": "module x"}, "A") + ctx := context.Background() + + partial := resolveCachesIdentity(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, "wf", "refs/heads/main", []models.CacheEntry{ + {Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}, + })[0] + if partial.Hash == "" { + t.Fatal("partial hash is empty, want the present file hashed") + } + if partial.RestoreKey != "" { + t.Fatalf("partial restore = %q, want a miss", partial.RestoreKey) + } + + none := resolveCachesIdentity(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, "wf", "refs/heads/main", []models.CacheEntry{ + {Key: "go-mod", Hash: []string{"nope.lock"}, Paths: []string{"/x"}}, + })[0] + if none.Hash != "" || none.RestoreKey != "" { + t.Fatalf("all-missing = hash %q, restore %q", none.Hash, none.RestoreKey) + } +} diff --git a/spindle/engine/cache_test.go b/spindle/engine/cache_test.go index 710ef3a1..426717e4 100644 --- a/spindle/engine/cache_test.go +++ b/spindle/engine/cache_test.go @@ -64,34 +64,6 @@ func (f *fakeStorage) has(key string) bool { return ok } -func gitRepo(t *testing.T, files map[string]string) (string, string) { - t.Helper() - if _, err := exec.LookPath("git"); err != nil { - t.Skip("git not available") - } - dir := t.TempDir() - run := func(args ...string) { - t.Helper() - cmd := exec.Command("git", append([]string{"-C", dir, "-c", "user.email=t@t", "-c", "user.name=t", "-c", "init.defaultBranch=main"}, args...)...) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - } - run("init") - for name, content := range files { - p := filepath.Join(dir, name) - if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(p, []byte(content), 0o644); err != nil { - t.Fatal(err) - } - } - run("add", ".") - run("commit", "-m", "init") - return dir, "HEAD" -} - func newCacheTestDB(t *testing.T) *db.DB { t.Helper() d, err := db.Make(context.Background(), filepath.Join(t.TempDir(), "spindle.db")) @@ -167,7 +139,7 @@ func TestCachePlanPreservesRestoreAtEntryQuota(t *testing.T) { "restore", repoDid.String(), "microvm/"+models.CacheNamespace(), - "deps", + cacheIdentityKey("", "", "deps"), "", "ready", time.Now(), @@ -283,95 +255,6 @@ func TestCachePlanKeepsPendingEntryThroughWorkflowDeadline(t *testing.T) { } } -func TestResolveCachesExactUnhashed(t *testing.T) { - ctx := context.Background() - d := newCacheTestDB(t) - base := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) - - insertCacheTestEntry(t, d, cacheTestEntry("exact", "did:plc:repo", "microvm", "deps", "", "ready", base)) - insertCacheTestEntry(t, d, cacheTestEntry("other-repo", "did:plc:other", "microvm", "deps", "", "ready", base.Add(time.Hour))) - insertCacheTestEntry(t, d, cacheTestEntry("other-engine", "did:plc:repo", "nixery", "deps", "", "ready", base.Add(time.Hour))) - insertCacheTestEntry(t, d, cacheTestEntry("pending", "did:plc:repo", "microvm", "deps", "", "pending", base.Add(time.Hour))) - insertCacheTestEntry(t, d, cacheTestEntry("hashed-only", "did:plc:repo", "microvm", "tools", "old", "ready", base)) - - resolved := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", "", "", []models.CacheEntry{ - {Key: "deps", Paths: []string{"/x"}, CompressionLevel: 7, When: "always"}, - {Key: "tools", Paths: []string{"/y"}}, - }) - if len(resolved) != 2 { - t.Fatalf("got %d resolved entries, want 2", len(resolved)) - } - got := resolved[0] - if got.RestoreID != "exact" || got.RestoreKey != "objects/exact" || got.RestoreName != "" { - t.Fatalf("exact restore = (%q, %q, %q)", got.RestoreID, got.RestoreKey, got.RestoreName) - } - if got.SaveKey != "" { - t.Fatalf("resolve allocated a save key %q", got.SaveKey) - } - if got.CompressionLevel != 7 || got.When != "always" { - t.Fatalf("resolved metadata = %+v", got) - } - if resolved[1].RestoreKey != "" { - t.Fatalf("unhashed entry used hashed fallback %q", resolved[1].RestoreKey) - } -} - -func TestResolveCachesHashExactAndFallback(t *testing.T) { - ctx := context.Background() - d := newCacheTestDB(t) - repoPath, rev := gitRepo(t, map[string]string{"go.sum": "v1 contents", "go.mod": "module x"}) - request := []models.CacheEntry{{Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}} - - first := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] - if first.Hash == "" { - t.Fatal("hash is empty") - } - - base := time.Date(2026, 4, 5, 6, 7, 8, 0, time.UTC) - insertCacheTestEntry(t, d, cacheTestEntry("exact", "did:plc:repo", "microvm", "go-mod", first.Hash, "ready", base.Add(time.Minute))) - insertCacheTestEntry(t, d, cacheTestEntry("sibling", "did:plc:repo", "microvm", "go-modules", "newer", "ready", base.Add(time.Hour))) - - exact := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] - if exact.RestoreID != "exact" || exact.RestoreKey != "objects/exact" || exact.RestoreName != "" { - t.Fatalf("exact generation restore = %+v", exact) - } - - if err := os.WriteFile(filepath.Join(repoPath, "go.sum"), []byte("v2 contents"), 0o644); err != nil { - t.Fatal(err) - } - commit := exec.Command("git", "-C", repoPath, "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qam", "bump") - if out, err := commit.CombinedOutput(); err != nil { - t.Fatalf("commit: %v\n%s", err, out) - } - rotated := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, request)[0] - if rotated.Hash == first.Hash { - t.Fatal("lockfile change did not rotate the cache hash") - } - if rotated.RestoreID != "exact" || rotated.RestoreKey != "objects/exact" || rotated.RestoreName != "go-mod-"+first.Hash { - t.Fatalf("fallback restore = %+v", rotated) - } -} - -func TestResolveCachesMissingHashFiles(t *testing.T) { - ctx := context.Background() - d := newCacheTestDB(t) - repoPath, rev := gitRepo(t, map[string]string{"go.mod": "module x"}) - - partial := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, []models.CacheEntry{ - {Key: "go-mod", Hash: []string{"go.sum", "go.mod"}, Paths: []string{"/x"}}, - })[0] - if len(partial.Hash) != 12 { - t.Fatalf("partial hash = %q, want 12 characters", partial.Hash) - } - - none := resolveCaches(ctx, discardLogger, d, "did:plc:repo", "microvm", repoPath, rev, []models.CacheEntry{ - {Key: "go-mod", Hash: []string{"nope.lock"}, Paths: []string{"/x"}}, - })[0] - if none.Hash != "" || none.RestoreKey != "" { - t.Fatalf("all-missing result = hash %q, restore %q", none.Hash, none.RestoreKey) - } -} - func TestTrackedCacheStoreSaveLifecycle(t *testing.T) { ctx := context.Background() d := newCacheTestDB(t)