From a3b8401e6cdfc13ce8bbd335cd10a9a48828fb79 Mon Sep 17 00:00:00 2001 From: Kieran Klukas Date: Tue, 24 Mar 2026 10:57:31 -0400 Subject: [PATCH] feat: add eye detection --- cull/CullApp.swift | 6 ++ cull/Models/CullSession.swift | 7 +- cull/Models/Photo.swift | 2 + cull/Services/QualityAnalyzer.swift | 80 ++++++++++++++-- cull/Services/WorkspaceDB.swift | 30 +++++- cull/Views/ContentView.swift | 143 +++++++++++++++++++++++++++- cull/Views/GroupDetailView.swift | 8 +- cull/Views/PhotoViewer.swift | 88 +++++++++++++++-- 8 files changed, 332 insertions(+), 32 deletions(-) diff --git a/cull/CullApp.swift b/cull/CullApp.swift index e911e8e..c43dd91 100644 --- a/cull/CullApp.swift +++ b/cull/CullApp.swift @@ -28,6 +28,11 @@ struct CullApp: App { .keyboardShortcut("e") .disabled(session.groups.isEmpty) + Button("Reanalyze Photos") { + NotificationCenter.default.post(name: .reimport, object: nil) + } + .disabled(session.sourceFolder == nil || session.isImporting) + Divider() Button("Close Folder") { @@ -126,4 +131,5 @@ struct CullApp: App { extension Notification.Name { static let openFolder = Notification.Name("openFolder") static let showExport = Notification.Name("showExport") + static let reimport = Notification.Name("reimport") } diff --git a/cull/Models/CullSession.swift b/cull/Models/CullSession.swift index a53c20e..c21db35 100644 --- a/cull/Models/CullSession.swift +++ b/cull/Models/CullSession.swift @@ -84,7 +84,6 @@ final class CullSession { func moveToNextGroup() { guard !groups.isEmpty else { return } - resetZoom() saveCursorPosition() let start = selectedGroupIndex for offset in 1...groups.count { @@ -99,7 +98,6 @@ final class CullSession { func moveToPreviousGroup() { guard !groups.isEmpty else { return } - resetZoom() saveCursorPosition() let start = selectedGroupIndex for offset in 1...groups.count { @@ -114,7 +112,6 @@ final class CullSession { func moveToNextPhoto() { guard let group = selectedGroup else { return } - resetZoom() // Try to find next visible photo in current group for i in (selectedPhotoIndex + 1).. FaceResult { guard let (source, imageIndex) = sourceForAnalysis(imageURL) else { - return FaceResult(sharpness: nil, regions: []) + return FaceResult(sharpness: nil, regions: [], eyeAspectRatios: []) } let options: [CFString: Any] = [ kCGImageSourceCreateThumbnailFromImageIfAbsent: true, @@ -132,15 +134,16 @@ struct QualityAnalyzer { kCGImageSourceCreateThumbnailWithTransform: true ] guard let cgImage = CGImageSourceCreateThumbnailAtIndex(source, imageIndex, options as CFDictionary) else { - return FaceResult(sharpness: nil, regions: []) + return FaceResult(sharpness: nil, regions: [], eyeAspectRatios: []) } - let request = VNDetectFaceCaptureQualityRequest() + let qualityRequest = VNDetectFaceCaptureQualityRequest() + let landmarksRequest = VNDetectFaceLandmarksRequest() let handler = VNImageRequestHandler(cgImage: cgImage, options: [:]) - try? handler.perform([request]) + try? handler.perform([qualityRequest, landmarksRequest]) - guard let results = request.results, !results.isEmpty else { - return FaceResult(sharpness: nil, regions: []) + guard let results = qualityRequest.results, !results.isEmpty else { + return FaceResult(sharpness: nil, regions: [], eyeAspectRatios: []) } // Filter out small background faces and low-confidence detections @@ -151,7 +154,7 @@ struct QualityAnalyzer { } guard !meaningful.isEmpty else { - return FaceResult(sharpness: nil, regions: []) + return FaceResult(sharpness: nil, regions: [], eyeAspectRatios: []) } // Sort faces by size (largest first) for better cycling order @@ -159,12 +162,29 @@ struct QualityAnalyzer { .map(\.boundingBox) .sorted { $0.width * $0.height > $1.width * $1.height } + // Build per-face EAR map keyed by bounding box + var earByBox: [CGRect: Double] = [:] + if let landmarkResults = landmarksRequest.results { + for face in landmarkResults { + if let landmarks = face.landmarks { + let leftEAR = eyeAspectRatio(landmarks.leftEye) + let rightEAR = eyeAspectRatio(landmarks.rightEye) + if let l = leftEAR, let r = rightEAR { + earByBox[face.boundingBox] = (l + r) / 2.0 + } + } + } + } + + // Map to sorted regions order + let eyeAspectRatios = regions.map { box in + earByBox.first { closeEnough($0.key, box) }?.value ?? 0.3 + } + // Measure sharpness directly on the largest face crop - // This is what actually matters — is the face in focus? let bestFaceRect = regions[0] let imageW = CGFloat(cgImage.width) let imageH = CGFloat(cgImage.height) - // Vision rect (bottom-left origin) → pixel rect (top-left origin), padded 20% let padX = bestFaceRect.width * 0.2 let padY = bestFaceRect.height * 0.2 let pixelRect = CGRect( @@ -180,7 +200,46 @@ struct QualityAnalyzer { faceSharpness = laplacianVariance(faceCrop) } - return FaceResult(sharpness: faceSharpness, regions: regions) + return FaceResult(sharpness: faceSharpness, regions: regions, eyeAspectRatios: eyeAspectRatios) + } + + /// Bounding boxes from different Vision requests may differ slightly + private static func closeEnough(_ a: CGRect, _ b: CGRect) -> Bool { + abs(a.origin.x - b.origin.x) < 0.01 && + abs(a.origin.y - b.origin.y) < 0.01 && + abs(a.width - b.width) < 0.01 && + abs(a.height - b.height) < 0.01 + } + + /// Eye Aspect Ratio (EAR) from Vision landmark points. + /// Uses vertical vs horizontal distances to detect closed eyes. + /// Returns nil if landmarks are unavailable. + private static func eyeAspectRatio(_ eye: VNFaceLandmarkRegion2D?) -> Double? { + guard let eye, eye.pointCount >= 6 else { return nil } + let pts = eye.normalizedPoints + // Vision eye landmarks: roughly ordered as outer corner, top points, inner corner, bottom points + // For 6-point eyes: 0=outer, 1=top-outer, 2=top-inner, 3=inner, 4=bottom-inner, 5=bottom-outer + // For 8-point eyes: 0=outer, 1=top-outer, 2=top, 3=top-inner, 4=inner, 5=bottom-inner, 6=bottom, 7=bottom-outer + let count = eye.pointCount + if count == 6 { + let vertical1 = distance(pts[1], pts[5]) + let vertical2 = distance(pts[2], pts[4]) + let horizontal = distance(pts[0], pts[3]) + guard horizontal > 0 else { return nil } + return Double((vertical1 + vertical2) / (2.0 * horizontal)) + } else if count >= 8 { + let vertical1 = distance(pts[1], pts[7]) + let vertical2 = distance(pts[2], pts[6]) + let vertical3 = distance(pts[3], pts[5]) + let horizontal = distance(pts[0], pts[4]) + guard horizontal > 0 else { return nil } + return Double((vertical1 + vertical2 + vertical3) / (3.0 * horizontal)) + } + return nil + } + + private static func distance(_ a: CGPoint, _ b: CGPoint) -> CGFloat { + hypot(a.x - b.x, a.y - b.y) } static func analyze(photo: Photo) async { @@ -193,6 +252,7 @@ struct QualityAnalyzer { photo.blurScore = blurResult photo.faceSharpness = faceResult.sharpness photo.faceRegions = faceResult.regions + photo.eyeAspectRatios = faceResult.eyeAspectRatios } } } diff --git a/cull/Services/WorkspaceDB.swift b/cull/Services/WorkspaceDB.swift index 8c8e628..a6a3ee2 100644 --- a/cull/Services/WorkspaceDB.swift +++ b/cull/Services/WorkspaceDB.swift @@ -17,6 +17,7 @@ final class WorkspaceDB: @unchecked Sendable { exec("PRAGMA synchronous=NORMAL") createTables() + migrate() } deinit { @@ -42,7 +43,8 @@ final class WorkspaceDB: @unchecked Sendable { paired_pixel_height INTEGER DEFAULT 0, paired_file_size INTEGER DEFAULT 0, capture_date REAL, - group_id TEXT + group_id TEXT, + eye_aspect_ratios TEXT ) """) @@ -61,6 +63,11 @@ final class WorkspaceDB: @unchecked Sendable { """) } + private func migrate() { + // Add eye_aspect_ratios column if missing (added in v2) + exec("ALTER TABLE photos ADD COLUMN eye_aspect_ratios TEXT") + } + // MARK: - Save func savePhotos(_ photos: [Photo], sourceFolder: URL) { @@ -69,8 +76,9 @@ final class WorkspaceDB: @unchecked Sendable { INSERT OR REPLACE INTO photos (path, paired_path, rating, flag, blur_score, face_sharpness, face_regions, pixel_width, pixel_height, file_size, - paired_pixel_width, paired_pixel_height, paired_file_size, capture_date, group_id) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) + paired_pixel_width, paired_pixel_height, paired_file_size, capture_date, group_id, + eye_aspect_ratios) + VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) """) defer { sqlite3_finalize(stmt) } @@ -96,6 +104,7 @@ final class WorkspaceDB: @unchecked Sendable { bind(stmt, 13, photo.pairedFileSize) bind(stmt, 14, photo.captureDate?.timeIntervalSinceReferenceDate) bind(stmt, 15, nil as String?) // group_id set separately + bind(stmt, 16, encodeDoubles(photo.eyeAspectRatios)) sqlite3_step(stmt) } exec("COMMIT") @@ -172,6 +181,7 @@ final class WorkspaceDB: @unchecked Sendable { let pairedFileSize: Int64 let captureDate: Date? let groupID: String? + let eyeAspectRatios: [Double] } func loadPhotos() -> [SavedPhoto] { @@ -195,6 +205,7 @@ final class WorkspaceDB: @unchecked Sendable { let pairedFileSize = sqlite3_column_int64(stmt, 12) let captureDateInterval = getOptionalDouble(stmt, 13) let groupID = getString(stmt, 14) + let earJSON = getString(stmt, 15) results.append(SavedPhoto( path: path, @@ -211,7 +222,8 @@ final class WorkspaceDB: @unchecked Sendable { pairedPixelHeight: pairedPixelHeight, pairedFileSize: pairedFileSize, captureDate: captureDateInterval.map { Date(timeIntervalSinceReferenceDate: $0) }, - groupID: groupID + groupID: groupID, + eyeAspectRatios: decodeDoubles(earJSON) )) } return results @@ -325,6 +337,16 @@ final class WorkspaceDB: @unchecked Sendable { return String(data: data, encoding: .utf8) } + private func encodeDoubles(_ values: [Double]) -> String? { + guard !values.isEmpty else { return nil } + return values.map { String(format: "%.3f", $0) }.joined(separator: ",") + } + + private func decodeDoubles(_ str: String?) -> [Double] { + guard let str, !str.isEmpty else { return [] } + return str.split(separator: ",").compactMap { Double($0) } + } + private func decodeRegions(_ json: String?) -> [CGRect] { guard let json, let data = json.data(using: .utf8), let arrays = try? JSONSerialization.jsonObject(with: data) as? [[Double]] else { return [] } diff --git a/cull/Views/ContentView.swift b/cull/Views/ContentView.swift index 300cc2b..1a681ba 100644 --- a/cull/Views/ContentView.swift +++ b/cull/Views/ContentView.swift @@ -37,6 +37,10 @@ struct ContentView: View { .onReceive(NotificationCenter.default.publisher(for: .showExport)) { _ in showExportSheet = true } + .onReceive(NotificationCenter.default.publisher(for: .reimport)) { _ in + guard let folder = session.sourceFolder else { return } + startReanalyze(folder) + } .onAppear { session.undoManager = windowUndoManager } @@ -230,6 +234,130 @@ struct ContentView: View { } } + @MainActor + private func startReanalyze(_ url: URL) { + session.isImporting = true + session.importProgress = 0.02 + cache.clearCache() + + // Snapshot existing ratings/flags keyed by relative path + let existingState: [String: (rating: Int, flag: PhotoFlag)] = { + var map: [String: (Int, PhotoFlag)] = [:] + for photo in session.allPhotos { + let rel = photo.url.relativePath(from: url) + map[rel] = (photo.rating, photo.flag) + } + return map + }() + + let s = session + let c = cache + + Task { + do { + // Phase 1: Full re-scan and metadata read + await MainActor.run { s.importStatus = "Scanning photos..." } + let result = try await PhotoImporter.importFolder(url, recursive: s.importRecursive) + + // Restore ratings/flags from previous state + for photo in result.photos { + let rel = photo.url.relativePath(from: url) + if let saved = existingState[rel] { + photo.rating = saved.rating + photo.flag = saved.flag + } + } + + // Phase 2: Re-group (0-20%) + await MainActor.run { s.importStatus = "Grouping similar shots..." } + var lastReported = 0.0 + let groups = await ShotGrouper.group(photos: result.photos) { p in + let mapped = p * 0.20 + guard mapped - lastReported > 0.01 else { return } + lastReported = mapped + await MainActor.run { + withAnimation(.linear(duration: 0.3)) { + s.importProgress = mapped + } + } + } + + // Phase 3: Analysis + Thumbnails + Previews in parallel (20-100%) + let allPhotos = groups.flatMap(\.photos) + await MainActor.run { s.importStatus = "Analyzing & loading..." } + + let totalPhotos = Double(allPhotos.count) + nonisolated(unsafe) var analysisProgress = 0.0 + nonisolated(unsafe) var thumbProgress = 0.0 + nonisolated(unsafe) var previewProgress = 0.0 + + @Sendable func reportProgress() async { + let combined = 0.20 + (analysisProgress * 0.40 + thumbProgress * 0.35 + previewProgress * 0.25) * 0.80 + await MainActor.run { + withAnimation(.linear(duration: 0.2)) { + s.importProgress = combined + } + } + } + + await withTaskGroup(of: Void.self) { parallelGroup in + parallelGroup.addTask { + var completed = 0.0 + for batchStart in stride(from: 0, to: allPhotos.count, by: 8) { + let batch = Array(allPhotos[batchStart.. $1.score }.map(\.photo) + } + + await MainActor.run { + s.importProgress = 1.0 + s.groups = groups + s.selectedGroupIndex = 0 + s.selectedPhotoIndex = 0 + s.isImporting = false + s.saveWorkspace() + } + } catch { + await MainActor.run { + s.isImporting = false + } + } + } + } + private var cullingView: some View { HStack(spacing: 0) { // Left: Groups column @@ -357,19 +485,28 @@ extension ContentView { /// Without faces: global blur score relative to group peers. static func qualityScore(_ photo: Photo, in group: PhotoGroup) -> Double { let peers = group.photos + var score: Double if let faceSharp = photo.faceSharpness, !photo.faceRegions.isEmpty { // Face detected — use face-region sharpness (Laplacian on face crop). // Normalize relative to peers who also have faces. let peerFaceScores = peers.compactMap(\.faceSharpness) if let maxF = peerFaceScores.max(), let minF = peerFaceScores.min(), maxF > minF { - return (faceSharp - minF) / (maxF - minF) + score = (faceSharp - minF) / (maxF - minF) + } else { + score = 0.5 } - return 0.5 } else { // No faces — use global blur score - return normalizedBlur(photo, peers: peers) + score = normalizedBlur(photo, peers: peers) } + + // Penalize photos with closed eyes + if photo.eyeAspectRatios.contains(where: { $0 < 0.20 }) { + score *= 0.3 + } + + return score } /// Normalize blur score relative to group peers (0-1 range) diff --git a/cull/Views/GroupDetailView.swift b/cull/Views/GroupDetailView.swift index e16a744..8101adb 100644 --- a/cull/Views/GroupDetailView.swift +++ b/cull/Views/GroupDetailView.swift @@ -69,8 +69,14 @@ private struct PhotoThumbnail: View { .font(.caption) } Spacer() + // Eyes closed badge + if photo.eyeAspectRatios.contains(where: { $0 < 0.20 }) { + Image(systemName: "eye.slash") + .foregroundStyle(.yellow) + .font(.caption) + } // Blur badge — hybrid: trust face quality for bokeh shots - if isPhotoBlurry() { + else if isPhotoBlurry() { Image(systemName: "eye.slash.fill") .foregroundStyle(.orange) .font(.caption) diff --git a/cull/Views/PhotoViewer.swift b/cull/Views/PhotoViewer.swift index 413c7bb..6ed5c46 100644 --- a/cull/Views/PhotoViewer.swift +++ b/cull/Views/PhotoViewer.swift @@ -120,6 +120,17 @@ struct PhotoViewer: View { } .font(.caption) + // Eyes closed badge + if photo.eyeAspectRatios.contains(where: { $0 < 0.20 }) { + let closedCount = photo.eyeAspectRatios.filter { $0 < 0.20 }.count + HStack(spacing: 3) { + Image(systemName: "eye.slash") + Text("\(closedCount)") + } + .foregroundStyle(.yellow) + .font(.caption) + } + // Blur badge if isPhotoBlurry(photo) { Label("Blurry", systemImage: "eye.slash.fill") @@ -286,16 +297,20 @@ struct PhotoViewer: View { return ZoomInfo(scale: 2.5, offset: .zero) } - guard photo.faceRegions.indices.contains(zoomIndex) else { - return ZoomInfo(scale: 1, offset: .zero) + // Zoomed to a face — adapt to current photo's faces + guard !photo.faceRegions.isEmpty else { + // No faces on this photo — fall back to center zoom + return ZoomInfo(scale: 2.5, offset: .zero) } - let faceRect = photo.faceRegions[zoomIndex] + // Clamp to available faces + let clampedIndex = min(zoomIndex, photo.faceRegions.count - 1) + let faceRect = photo.faceRegions[clampedIndex] // Vision coordinates: origin bottom-left, normalized 0-1 // Calculate scale so the face takes up ~35% of the view width let faceW = faceRect.width let faceH = faceRect.height - let scale = min(0.35 / max(faceW, faceH), 5.0) + let scale = min(max(0.35 / max(faceW, faceH), 1.5), 5.0) // Face center in normalized image coords (flip Y) let faceCenterX = faceRect.midX @@ -316,6 +331,55 @@ struct PhotoViewer: View { return ZoomInfo(scale: scale, offset: CGSize(width: offsetX, height: offsetY)) } + // MARK: - Eye indicator + + /// Draws two small eye shapes that reflect how open/closed the eyes are. + /// EAR ~0.30 = wide open, ~0.20 = threshold, ~0.05 = shut. + private struct EyeIndicator: View { + let ear: Double + let faceWidth: CGFloat + + var body: some View { + let eyeW = min(faceWidth * 0.22, 20) + // Map EAR to openness: 0.05→flat, 0.30→full open + let openness = CGFloat(max(0, min(1, (ear - 0.05) / 0.25))) + let eyeH = eyeW * 0.8 * openness + let color: Color = ear < 0.20 ? .yellow : .white.opacity(0.7) + + EyeShape(openness: openness) + .fill(color.opacity(0.9)) + .frame(width: eyeW, height: max(eyeH, 1.5)) + .shadow(color: .black.opacity(0.8), radius: 1.5) + } + } + + /// Almond-shaped eye that flattens as openness approaches 0. + private struct EyeShape: Shape { + let openness: CGFloat + + func path(in rect: CGRect) -> Path { + var path = Path() + let midY = rect.midY + let bulge = rect.height / 2 + let cpInset = rect.width * 0.2 + + // Top lid arc (cubic for rounder shape) + path.move(to: CGPoint(x: rect.minX, y: midY)) + path.addCurve( + to: CGPoint(x: rect.maxX, y: midY), + control1: CGPoint(x: rect.minX + cpInset, y: midY - bulge), + control2: CGPoint(x: rect.maxX - cpInset, y: midY - bulge) + ) + // Bottom lid arc + path.addCurve( + to: CGPoint(x: rect.minX, y: midY), + control1: CGPoint(x: rect.maxX - cpInset, y: midY + bulge), + control2: CGPoint(x: rect.minX + cpInset, y: midY + bulge) + ) + return path + } + } + private func fittedSize(image: CGSize, in container: CGSize) -> CGSize { let scaleW = container.width / image.width let scaleH = container.height / image.height @@ -329,16 +393,24 @@ struct PhotoViewer: View { private func faceOverlays(photo: Photo, fittedSize: CGSize) -> some View { ForEach(0..