From 01100e58855847438dadf0a3b34210b1aeb26f22 Mon Sep 17 00:00:00 2001 From: Corbin Crutchley Date: Thu, 17 Sep 2026 16:18:35 -0700 Subject: [PATCH] feat: add cancellable Slang compilation and dependency caching --- ScreenSlanger/slang_compiler.swift | 299 +++++++++++++----- ScreenSlangerTests/CompilerProcessTests.swift | 151 +++++++++ ShaderCompilerProbe/main.swift | 4 + 3 files changed, 380 insertions(+), 74 deletions(-) diff --git a/ScreenSlanger/slang_compiler.swift b/ScreenSlanger/slang_compiler.swift index 12b51e9..c1026a0 100644 --- a/ScreenSlanger/slang_compiler.swift +++ b/ScreenSlanger/slang_compiler.swift @@ -1,4 +1,7 @@ +import CryptoKit +import Darwin import Foundation +import Synchronization /// Errors that can occur during Slang shader compilation enum SlangCompilerError: Error, LocalizedError { @@ -6,6 +9,7 @@ enum SlangCompilerError: Error, LocalizedError { case compilationFailed(String) case invalidOutput case processError(String) + case timedOut(TimeInterval) var errorDescription: String? { switch self { @@ -17,13 +21,15 @@ enum SlangCompilerError: Error, LocalizedError { return "Slang compiler produced invalid output" case .processError(let message): return "Process error: \(message)" + case .timedOut(let seconds): + return "Shader compilation exceeded \(seconds) seconds" } } } /// Represents a shader parameter parsed from #pragma parameter directives /// Format: #pragma parameter NAME "Description" default min max step -struct ShaderParameter { +struct ShaderParameter: Sendable { let name: String let description: String let defaultValue: Float @@ -73,62 +79,162 @@ struct ShaderParameter { } /// Result of preprocessing a RetroArch-style shader -struct PreprocessedShader { +struct PreprocessedShader: Sendable { let source: String let parameters: [ShaderParameter] let isRetroArchStyle: Bool } -/// Capture both output streams on disk so verbose compiler diagnostics cannot fill a pipe -/// while the parent process waits for compilation to finish. -func runShaderCompilerProcess(_ process: Process, outputFile: URL) throws -> String { +/// Shared between a UI task and its synchronous compiler worker. The compiler owns +/// its Process, so cancellation never touches Foundation process state across threads. +final class ShaderCompilationCancellation: Sendable { + private let cancelled = Mutex(false) + + func cancel() { + cancelled.withLock { $0 = true } + } + + func checkCancellation() throws { + if cancelled.withLock({ $0 }) { throw CancellationError() } + } +} + +/// Capture both output streams on disk so verbose diagnostics cannot fill a pipe. +/// Call from a worker: a bounded poll makes cancellation independent of compiler output. +func runShaderCompilerProcess( + _ process: Process, + outputFile: URL, + cancellation: ShaderCompilationCancellation? = nil, + timeout: TimeInterval = 30 +) throws -> String { + try cancellation?.checkCancellation() + guard timeout.isFinite, timeout >= 0 else { + throw SlangCompilerError.processError("Compiler timeout must be a finite, nonnegative duration") + } try Data().write(to: outputFile) let outputHandle = try FileHandle(forWritingTo: outputFile) defer { try? outputHandle.close() } - process.standardOutput = outputHandle process.standardError = outputHandle try process.run() - process.waitUntilExit() + let clock = ContinuousClock() + let deadline = clock.now.advanced(by: .seconds(max(0, timeout))) + do { + while process.isRunning { + try cancellation?.checkCancellation() + if clock.now >= deadline { throw SlangCompilerError.timedOut(timeout) } + Thread.sleep(forTimeInterval: 0.01) + } + process.waitUntilExit() + try cancellation?.checkCancellation() + } catch { + if process.isRunning { + process.terminate() + let grace = clock.now.advanced(by: .milliseconds(100)) + while process.isRunning && clock.now < grace { + Thread.sleep(forTimeInterval: 0.005) + } + // A compiler can ignore SIGTERM. Always reap it before removing its files. + if process.isRunning { kill(process.processIdentifier, SIGKILL) } + } + process.waitUntilExit() + throw error + } return String(decoding: try Data(contentsOf: outputFile), as: UTF8.self) } /// Wrapper for the Slang shader compiler class SlangCompiler { - /// Possible locations to search for slangc - private static let slangcSearchPaths = [ - // Environment variable override - ProcessInfo.processInfo.environment["SLANG_PATH"], - // Version managed by scripts/setup-dependencies.sh (also works when launched from Finder). - FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent("Library/Application Support/ScreenSlanger/Tools/slang/current/bin/slangc").path, - // Homebrew installation - "/opt/homebrew/bin/slangc", - "/usr/local/bin/slangc", - // App bundle resources - Bundle.main.path(forResource: "slangc", ofType: nil), - // Development paths - "/usr/local/slang/bin/slangc", - // User home directory - FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent(".slang/bin/slangc").path, - // Common Slang download locations - FileManager.default.homeDirectoryForCurrentUser - .appendingPathComponent("slang/bin/slangc").path - ].compactMap { $0 } - - /// Find the slangc executable + /// Read the override on each request, and resolve managed installation symlinks + /// in the cache identity so an upgraded compiler never reuses an old artifact. static func findSlangc() -> String? { - for path in slangcSearchPaths { - if FileManager.default.isExecutableFile(atPath: path) { - return path + let home = FileManager.default.homeDirectoryForCurrentUser + let paths = [ + ProcessInfo.processInfo.environment["SLANG_PATH"], + home.appendingPathComponent("Library/Application Support/ScreenSlanger/Tools/slang/current/bin/slangc").path, + "/opt/homebrew/bin/slangc", + "/usr/local/bin/slangc", + Bundle.main.path(forResource: "slangc", ofType: nil), + "/usr/local/slang/bin/slangc", + home.appendingPathComponent(".slang/bin/slangc").path, + home.appendingPathComponent("slang/bin/slangc").path + ].compactMap { $0 } + return paths.first { FileManager.default.isExecutableFile(atPath: $0) } + } + + private struct CachedCompilation: Sendable { + let metalSource: String + let dependencies: [URL: Data] + let searchDirectories: [URL: Date] + } + + private static let cache = Mutex<[String: CachedCompilation]>([:]) + + static func invalidateCache() { + cache.withLock { $0.removeAll() } + } + + private static func digest(_ data: Data) -> Data { Data(SHA256.hash(data: data)) } + + /// Search directory timestamps catch newly added files that can shadow a previous + /// include/import. Dependency contents (not timestamps) catch edits to existing files. + private static func directorySnapshot(_ roots: [URL]) -> [URL: Date] { + var snapshot: [URL: Date] = [:] + for root in roots { + if let date = try? root.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate { + snapshot[root] = date + } + guard let directories = FileManager.default.enumerator( + at: root, includingPropertiesForKeys: [.isDirectoryKey, .contentModificationDateKey], + options: []) else { continue } + for case let directory as URL in directories { + if directory.lastPathComponent == ".git" { directories.skipDescendants(); continue } + if let values = try? directory.resourceValues(forKeys: [.isDirectoryKey, .contentModificationDateKey]), + values.isDirectory == true, let date = values.contentModificationDate { + snapshot[directory] = date + } } } - return nil + return snapshot } - + + /// Parse the compiler's Make-style dependency file, including escaped paths. + /// Asking slangc is essential: regex scanning misses conditional includes and imports. + static func dependencyPaths(from depfile: String, relativeTo directory: URL) -> [URL] { + var escaped = false + var foundTarget = false + var token = "" + var paths: [String] = [] + for character in depfile { + if escaped { + if character != "\n" && character != "\r" { token.append(character) } + escaped = false + } else if character == "\\" { + escaped = true + } else if !foundTarget && character == ":" { + foundTarget = true + token = "" + } else if character.isWhitespace { + if foundTarget && !token.isEmpty { paths.append(token) } + token = "" + } else { + token.append(character) + } + } + if foundTarget && !token.isEmpty { paths.append(token) } + return paths.map { path in + URL(fileURLWithPath: path.replacingOccurrences(of: "$$", with: "$"), relativeTo: directory) + .standardizedFileURL.resolvingSymlinksInPath() + } + } + + private static func escapedLinePath(_ url: URL) -> String { + url.path.replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + } + /// Check if Slang compiler is available static var isAvailable: Bool { return findSlangc() != nil @@ -143,65 +249,108 @@ class SlangCompiler { static func compileToMetal( slangSource: String, entryPoint: String = "fragmentMain", - stage: String = "fragment" + stage: String = "fragment", + sourceURL: URL? = nil, + includeDirectories: [URL] = [], + cancellation: ShaderCompilationCancellation? = nil, + timeout: TimeInterval = 30 ) throws -> String { - guard let slangcPath = findSlangc() else { - throw SlangCompilerError.slangcNotFound + try cancellation?.checkCancellation() + guard let slangcPath = findSlangc() else { throw SlangCompilerError.slangcNotFound } + let compilerURL = URL(fileURLWithPath: slangcPath).resolvingSymlinksInPath() + let attributes = try FileManager.default.attributesOfItem(atPath: compilerURL.path) + // Native distributions keep their compiler libraries adjacent to bin/. Including + // that directory's identity also invalidates updates that only replace a dylib. + let libraryDirectory = compilerURL.deletingLastPathComponent().deletingLastPathComponent().appendingPathComponent("lib") + let libraryFiles = (try? FileManager.default.contentsOfDirectory(at: libraryDirectory, + includingPropertiesForKeys: [.contentModificationDateKey, .fileSizeKey])) ?? [] + let libraryIdentity = libraryFiles.sorted { $0.path < $1.path }.map { url in + let values = try? url.resourceValues(forKeys: [.contentModificationDateKey, .fileSizeKey]) + return "\(url.path):\(values?.contentModificationDate?.timeIntervalSince1970 ?? 0):\(values?.fileSize ?? 0)" + }.joined(separator: "|") + var roots = ([sourceURL?.deletingLastPathComponent()].compactMap { $0 } + includeDirectories) + .map { $0.standardizedFileURL.resolvingSymlinksInPath() } + var seen = Set() + roots = roots.filter { seen.insert($0).inserted } + let keyParts = [slangSource, entryPoint, stage, sourceURL?.path ?? "", compilerURL.path, + String(describing: attributes[.modificationDate]), String(describing: attributes[.size]), + libraryIdentity] + roots.map(\.path) + let key = digest(try JSONEncoder().encode(keyParts)).base64EncodedString() + let directories = directorySnapshot(roots) + if let cached = cache.withLock({ $0[key] }), cached.searchDirectories == directories, + cached.dependencies.allSatisfy({ url, fingerprint in + (try? Data(contentsOf: url)).map(digest) == fingerprint + }) { + try cancellation?.checkCancellation() + return cached.metalSource } - - // Create a temporary directory for compilation - let tempDir = FileManager.default.temporaryDirectory - .appendingPathComponent(UUID().uuidString) + // Evict stale successes before starting a replacement, including failed retries. + _ = cache.withLock { $0.removeValue(forKey: key) } + let startedAt = Date() + let tempDir = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) - - defer { - try? FileManager.default.removeItem(at: tempDir) - } - - // Write the Slang source to a temporary file + defer { try? FileManager.default.removeItem(at: tempDir) } let inputFile = tempDir.appendingPathComponent("shader.slang") let outputFile = tempDir.appendingPathComponent("shader.metal") - - try slangSource.write(to: inputFile, atomically: true, encoding: .utf8) - - // Build the slangc command + let depfile = tempDir.appendingPathComponent("shader.d") + let source = sourceURL.map { "#line 1 \"\(escapedLinePath($0))\"\n" + slangSource } ?? slangSource + try source.write(to: inputFile, atomically: true, encoding: .utf8) let process = Process() - process.executableURL = URL(fileURLWithPath: slangcPath) - process.arguments = [ - inputFile.path, - "-target", "metal", - "-entry", entryPoint, - "-stage", stage, - "-o", outputFile.path - ] - + process.executableURL = compilerURL + process.currentDirectoryURL = sourceURL?.deletingLastPathComponent() ?? tempDir + process.arguments = [inputFile.path, "-target", "metal", "-entry", entryPoint, + "-stage", stage, "-o", outputFile.path, "-depfile", depfile.path] + + roots.flatMap { ["-I", $0.path] } let diagnostics: String do { - diagnostics = try runShaderCompilerProcess( - process, outputFile: tempDir.appendingPathComponent("slangc.log")) + diagnostics = try runShaderCompilerProcess(process, + outputFile: tempDir.appendingPathComponent("slangc.log"), + cancellation: cancellation, timeout: timeout) + } catch is CancellationError { + throw CancellationError() + } catch let error as SlangCompilerError { + throw error } catch { throw SlangCompilerError.processError(error.localizedDescription) } - - // Check for compilation errors - if process.terminationStatus != 0 { + guard process.terminationStatus == 0 else { throw SlangCompilerError.compilationFailed( - diagnostics.isEmpty ? "slangc exited with code \(process.terminationStatus)" : diagnostics - ) + diagnostics.isEmpty ? "slangc exited with code \(process.terminationStatus)" : diagnostics) } - - // Read the generated Metal source - guard let metalSource = try? String(contentsOf: outputFile, encoding: .utf8) else { + guard let metalSource = try? String(contentsOf: outputFile, encoding: .utf8), !metalSource.isEmpty else { throw SlangCompilerError.invalidOutput } - + try cancellation?.checkCancellation() + if let dependencies = try? String(contentsOf: depfile, encoding: .utf8) { + let paths = dependencyPaths(from: dependencies, relativeTo: process.currentDirectoryURL ?? tempDir) + .filter { $0 != inputFile.resolvingSymlinksInPath() } + var fingerprints: [URL: Data] = [:] + var stable = true + for url in paths { + guard let data = try? Data(contentsOf: url), + let date = try? url.resourceValues(forKeys: [.contentModificationDateKey]).contentModificationDate, + date <= startedAt else { stable = false; break } + fingerprints[url] = digest(data) + } + // Do not cache a compile raced by filesystem changes or an incomplete depfile. + if stable && !dependencies.isEmpty && directorySnapshot(roots) == directories { + let entry = CachedCompilation(metalSource: metalSource, dependencies: fingerprints, + searchDirectories: directories) + cache.withLock { + if $0.count >= 32 { $0.removeAll(keepingCapacity: true) } + $0[key] = entry + } + } + } return metalSource } - + /// Wrap user-provided Slang effect code with the ScreenSlanger framework code /// This creates a complete Slang shader that matches ScreenSlanger's expectations - static func wrapEffectSource(_ effectSource: String) -> String { + static func wrapEffectSource(_ effectSource: String, sourceURL: URL? = nil) -> String { + let origin = sourceURL.map { "#line 1 \"\(escapedLinePath($0))\"" } ?? "#line 1 \"effect.slang\"" return """ + #line 1 "ScreenSlanger-wrapper.slang" // ScreenSlanger Slang wrapper // This shader is compiled from Slang to Metal @@ -249,7 +398,9 @@ class SlangCompiler { } // ========== USER EFFECT CODE BEGIN ========== + \(origin) \(effectSource) + #line 1 "ScreenSlanger-wrapper.slang" // ========== USER EFFECT CODE END ========== // Fragment shader output diff --git a/ScreenSlangerTests/CompilerProcessTests.swift b/ScreenSlangerTests/CompilerProcessTests.swift index 8a93146..df8eb93 100644 --- a/ScreenSlangerTests/CompilerProcessTests.swift +++ b/ScreenSlangerTests/CompilerProcessTests.swift @@ -42,3 +42,154 @@ struct CompilerProcessTests { #expect(process.terminationStatus == 0, "Large-output compiler probe failed: \(diagnostic)") } } + + +extension CompilerProcessTests { + private func withHangingCompiler( + _ body: (Process, URL) throws -> Void + ) throws { + let products = Bundle(for: CompilerProbeBundle.self).bundleURL.deletingLastPathComponent() + let executable = products.appendingPathComponent("ShaderCompilerProbe") + try #require(FileManager.default.isExecutableFile(atPath: executable.path)) + let temporary = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + try FileManager.default.createDirectory(at: temporary, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: temporary) } + let process = Process() + process.executableURL = executable + process.arguments = ["--hang"] + try body(process, temporary.appendingPathComponent("compiler.log")) + } + + @Test("A hung compiler is killed and reaped within its deadline") + func compilerTimeout() throws { + try withHangingCompiler { process, log in + let start = ContinuousClock.now + do { + _ = try runShaderCompilerProcess(process, outputFile: log, timeout: 0.3) + Issue.record("Hung compiler unexpectedly completed") + } catch SlangCompilerError.timedOut { + #expect(!process.isRunning) + #expect(process.terminationReason == .uncaughtSignal) + #expect(ContinuousClock.now - start < .seconds(3)) + } + } + } + + @Test("Cancellation stops an already running compiler") + func compilerCancellation() throws { + let cancellation = ShaderCompilationCancellation() + DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { cancellation.cancel() } + try withHangingCompiler { process, log in + let start = ContinuousClock.now + do { + _ = try runShaderCompilerProcess(process, outputFile: log, + cancellation: cancellation, timeout: 5) + Issue.record("Cancelled compiler unexpectedly completed") + } catch is CancellationError { + #expect(!process.isRunning) + #expect(ContinuousClock.now - start < .seconds(3)) + } + } + } + + @Test("An already cancelled request does not launch a compiler") + func cancelledBeforeLaunch() throws { + let cancellation = ShaderCompilationCancellation() + cancellation.cancel() + try withHangingCompiler { process, log in + #expect(throws: CancellationError.self) { + try runShaderCompilerProcess(process, outputFile: log, cancellation: cancellation) + } + #expect(process.processIdentifier == 0) + #expect(!FileManager.default.fileExists(atPath: log.path)) + } + } +} + +@Suite(.serialized) +struct NativeCompilerTests { + private func withDirectory(_ body: (URL) throws -> Void) throws { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("Slang fixture \(UUID().uuidString)") + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: directory) } + try body(directory) + } + + @Test("Native imports track transitive includes and invalidate cached output") + func importAndTransitiveIncludeChanges() throws { + try #require(SlangCompiler.isAvailable, "Install the compiler with scripts/setup-dependencies.sh") + try withDirectory { directory in + let effect = directory.appendingPathComponent("effect.slang") + let module = directory.appendingPathComponent("gain.slang") + let include = directory.appendingPathComponent("nested header.slangh") + try "float gain() { return 0.25; }".write(to: include, atomically: true, encoding: .utf8) + try "module gain;\n#include \"nested header.slangh\"\npublic float moduleGain() { return gain(); }" + .write(to: module, atomically: true, encoding: .utf8) + let source = "import gain;\nfloat4 shaderFunction(ShaderInput input) { return float4(moduleGain(),0,0,1); }" + try source.write(to: effect, atomically: true, encoding: .utf8) + let wrapped = SlangCompiler.wrapEffectSource(source, sourceURL: effect) + func compile() throws -> String { + try SlangCompiler.compileToMetal(slangSource: wrapped, sourceURL: effect) + } + let first = try compile() + #expect(first.contains("0.25f")) + #expect(try compile() == first) + // An atomic save replaces the included file without changing the root source. + try "float gain() { return 0.75; }".write(to: include, atomically: true, encoding: .utf8) + let changed = try compile() + #expect(changed.contains("0.75f")) + #expect(changed != first) + // Failed replacements cannot fall back to an older successful artifact. + try "float gain() { return missingIncludedSymbol; }".write(to: include, atomically: true, encoding: .utf8) + for _ in 0..<2 { + do { + _ = try compile() + Issue.record("Invalid included shader returned a stale cached result") + } catch SlangCompilerError.compilationFailed(let diagnostic) { + #expect(diagnostic.contains("missingIncludedSymbol")) + #expect(diagnostic.contains("nested header.slangh")) + } + } + try "float gain() { return 0.5; }".write(to: include, atomically: true, encoding: .utf8) + #expect(try compile().contains("0.5f")) + } + } + + @Test("Native include search roots and original diagnostic line numbers are preserved") + func includeRootsAndDiagnostics() throws { + try #require(SlangCompiler.isAvailable) + try withDirectory { directory in + let sourceDirectory = directory.appendingPathComponent("shaders") + let includes = directory.appendingPathComponent("includes") + try FileManager.default.createDirectory(at: sourceDirectory, withIntermediateDirectories: true) + try FileManager.default.createDirectory(at: includes, withIntermediateDirectories: true) + try "float4 effectColor() { return float4(0,1,0,1); }" + .write(to: includes.appendingPathComponent("color.slangh"), atomically: true, encoding: .utf8) + let effect = sourceDirectory.appendingPathComponent("original effect.slang") + let source = "#include \"color.slangh\"\nfloat4 shaderFunction(ShaderInput input) { return effectColor(); }" + let output = try SlangCompiler.compileToMetal( + slangSource: SlangCompiler.wrapEffectSource(source, sourceURL: effect), + sourceURL: effect, includeDirectories: [includes]) + #expect(output.contains("effectColor")) + do { + _ = try SlangCompiler.compileToMetal(slangSource: SlangCompiler.wrapEffectSource( + "// original line one\nfloat4 shaderFunction(ShaderInput input) { return missingOriginalSymbol; }", + sourceURL: effect), sourceURL: effect) + Issue.record("Invalid shader compiled") + } catch SlangCompilerError.compilationFailed(let diagnostic) { + #expect(diagnostic.contains(effect.path + ":2:") || diagnostic.contains(effect.path + "(2)")) + #expect(diagnostic.contains("missingOriginalSymbol")) + } + } + } + + @Test("Compiler dependency paths preserve escaped spaces, hashes, dollars and continuations") + func makeDependencyPaths() { + let directory = URL(fileURLWithPath: "/tmp/dependencies") + let paths = SlangCompiler.dependencyPaths( + from: "shader.metal: nested\\ header.slangh escaped\\#name.slang dollar$$name.slang \\\n module.slang\n", + relativeTo: directory) + #expect(paths.map(\.lastPathComponent) == ["nested header.slangh", "escaped#name.slang", "dollar$name.slang", "module.slang"]) + } +} diff --git a/ShaderCompilerProbe/main.swift b/ShaderCompilerProbe/main.swift index e3b709b..4487d58 100644 --- a/ShaderCompilerProbe/main.swift +++ b/ShaderCompilerProbe/main.swift @@ -30,6 +30,10 @@ if CommandLine.arguments.dropFirst().first == "--verify" { } catch { fail("Unexpected compiler failure: \(error.localizedDescription)") } +} else if CommandLine.arguments.dropFirst().first == "--hang" { + // Exercise forced termination, not only the happy path where SIGTERM works. + signal(SIGTERM, SIG_IGN) + while true { Thread.sleep(forTimeInterval: 1) } } else { // The production compiler invokes this executable as its mock slangc. // Both streams exceed a pipe's capacity, exposing wait-before-drain bugs. -- 2.51.2