diff --git a/scripts/find-video-duplicates.rcsh b/scripts/find-video-duplicates.rcsh new file mode 100755 index 0000000..ef42944 --- /dev/null +++ b/scripts/find-video-duplicates.rcsh @@ -0,0 +1,384 @@ +#!/usr/bin/env rc + +flag e + + +mode=print +root=. +out=duplicates +threshold=0.080 +high_threshold=0.025 +duration_slop=1.0 + +fn usage { + echo 'usage: find-video-duplicates.rcsh [print|links] [--out dir] [--threshold n] [--high-threshold n] [--duration-slop seconds] [folder]' >[1=2] + echo print: write newline-delimited deletion candidates to stdout >[1=2] + echo links: create duplicate review folders with relative symlinks >[1=2] +} + +while(! ~ $1 ()) { + switch($1) { + case print list + mode=print + shift + case links symlinks review + mode=links + shift + case --out + if(~ $#* 1) { + usage + exit 2 + } + out=$2 + shift + shift + case --threshold + if(~ $#* 1) { + usage + exit 2 + } + threshold=$2 + shift + shift + case --high-threshold + if(~ $#* 1) { + usage + exit 2 + } + high_threshold=$2 + shift + shift + case --duration-slop + if(~ $#* 1) { + usage + exit 2 + } + duration_slop=$2 + shift + shift + case -h --help + usage + exit 0 + case * + if(! ~ $root .) { + echo Unexpected argument: $1 >[1=2] + usage + exit 2 + } + root=$1 + shift + } +} + +if(! test -d $root) { + echo Not a directory: $root >[1=2] + exit 1 +} + +js=`{mktemp -t find-video-duplicates.XXXXXX.js} + +fn cleanup { + rm -f $js +} + +fn sigexit { + cleanup +} + +fn sigint { + cleanup + exit interrupted +} + +cat > $js <<'js' +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; + +const [mode, rootArg, outArg, thresholdArg, highThresholdArg, durationSlopArg] = Bun.argv.slice(2); +const root = path.resolve(rootArg); +const out = outArg; +const threshold = Number(thresholdArg); +const highThreshold = Number(highThresholdArg); +const durationSlop = Number(durationSlopArg); +const videoExts = new Set([ + ".3g2", + ".3gp", + ".avi", + ".flv", + ".m2ts", + ".m4v", + ".mkv", + ".mov", + ".mp4", + ".mpeg", + ".mpg", + ".mts", + ".ogm", + ".ogv", + ".ts", + ".webm", + ".wmv", +]); + +class UnionFind { + constructor(count) { + this.parent = Array.from({ length: count }, (_, index) => index); + } + + find(value) { + while(this.parent[value] !== value) { + this.parent[value] = this.parent[this.parent[value]]; + value = this.parent[value]; + } + return value; + } + + union(left, right) { + const leftRoot = this.find(left); + const rightRoot = this.find(right); + if(leftRoot !== rightRoot) + this.parent[rightRoot] = leftRoot; + } +} + +function run(command) { + return spawnSync(command[0], command.slice(1), { encoding: "utf8" }); +} + +function nixRun(attr, args) { + return run(["nix", "run", `nixpkgs#${attr}`, "--", ...args]); +} + +function creationTime(stat) { + return stat.birthtimeMs || stat.ctimeMs; +} + +function ffmpegDuration(file) { + const result = nixRun("ffmpeg", ["-hide_banner", "-i", file]); + const match = (result.stderr || "").match(/Duration: (\d+):(\d+):(\d+(?:\.\d+)?)/); + if(!match) { + console.error(`skip: no duration: ${file}`); + return null; + } + const [, hours, minutes, seconds] = match; + return Number(hours) * 3600 + Number(minutes) * 60 + Number(seconds); +} + +function makeSheet(file, duration, sheet) { + const fps = Math.min(2.0, Math.max(0.05, 25.0 / duration)); + const vf = [ + `fps=${fps.toFixed(6)}`, + "scale=96:96:force_original_aspect_ratio=decrease", + "pad=96:96:(ow-iw)/2:(oh-ih)/2", + "format=gray", + "tile=5x5", + ].join(","); + const result = nixRun("ffmpeg", [ + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + file, + "-vf", + vf, + "-frames:v", + "1", + sheet, + ]); + if(result.status !== 0 || !fs.existsSync(sheet)) { + console.error(`skip: thumbnailing failed: ${file}`); + if(result.stderr) + console.error(result.stderr.trim()); + return false; + } + return true; +} + +function rmse(left, right) { + const result = nixRun("imagemagick", [ + "compare", + "-metric", + "RMSE", + left, + right, + "null:", + ]); + const text = result.stderr || ""; + const match = text.match(/\(([-+0-9.eE]+)\)/); + if(!match) { + console.error(`warn: could not parse compare output: ${text.trim()}`); + return null; + } + return Number(match[1]); +} + +function walkVideos(directory, result = []) { + const reviewRoot = path.resolve(root, out); + for(const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const file = path.join(directory, entry.name); + if(entry.isDirectory()) { + if(mode === "links" && path.resolve(file) === reviewRoot) + continue; + walkVideos(file, result); + } else if(entry.isFile() && videoExts.has(path.extname(entry.name).toLowerCase())) { + result.push(file); + } + } + return result.sort(); +} + +function edgeKey(left, right) { + return [left, right].sort((a, b) => a - b).join(":"); +} + +function keepScore(group, edges, video) { + let highConfidence = false; + for(const other of group) { + if(other.index === video.index) + continue; + const score = edges.get(edgeKey(video.index, other.index)); + if(score !== undefined && score <= highThreshold) { + highConfidence = true; + break; + } + } + const oldBonus = highConfidence ? -video.ctime : 0; + return [video.size, oldBonus, -video.index, video.file]; +} + +function compareScore(left, right) { + for(let index = 0; index < left.length; index++) { + if(left[index] < right[index]) + return -1; + if(left[index] > right[index]) + return 1; + } + return 0; +} + +function bestKeep(group, edges) { + return group.reduce((best, video) => { + if(!best) + return video; + return compareScore(keepScore(group, edges, video), keepScore(group, edges, best)) > 0 ? video : best; + }, null); +} + +function relativeLinkTarget(link, target) { + return path.relative(path.dirname(link), target); +} + +function uniqueLinkName(directory, target, seen) { + const parsed = path.parse(target); + const digest = crypto.createHash("sha1").update(target).digest("hex").slice(0, 8); + const candidates = [ + `${parsed.name}${parsed.ext}`, + `${parsed.name}-${digest}${parsed.ext}`, + ]; + for(const candidate of candidates) { + if(!seen.has(candidate) && !fs.existsSync(path.join(directory, candidate))) { + seen.add(candidate); + return candidate; + } + } + for(let counter = 2;; counter++) { + const candidate = `${parsed.name}-${digest}-${counter}${parsed.ext}`; + if(!seen.has(candidate) && !fs.existsSync(path.join(directory, candidate))) { + seen.add(candidate); + return candidate; + } + } +} + +const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "video-dups.")); + +try { + const videos = []; + for(const [index, file] of walkVideos(root).entries()) { + const stat = fs.statSync(file); + const duration = ffmpegDuration(file); + if(!Number.isFinite(duration) || duration <= 0) + continue; + const sheet = path.join(tmp, `${index}.png`); + if(!makeSheet(file, duration, sheet)) + continue; + videos.push({ + index, + file, + size: stat.size, + ctime: creationTime(stat), + duration, + sheet, + }); + } + + const uf = new UnionFind(videos.length); + const edges = new Map(); + + for(let leftIndex = 0; leftIndex < videos.length; leftIndex++) { + const left = videos[leftIndex]; + for(let rightIndex = leftIndex + 1; rightIndex < videos.length; rightIndex++) { + const right = videos[rightIndex]; + if(Math.abs(left.duration - right.duration) > durationSlop) + continue; + const score = rmse(left.sheet, right.sheet); + if(score === null) + continue; + if(score <= threshold) { + uf.union(leftIndex, rightIndex); + edges.set(edgeKey(left.index, right.index), score); + } + } + } + + const groupsByRoot = new Map(); + for(const [index, video] of videos.entries()) { + const groupRoot = uf.find(index); + if(!groupsByRoot.has(groupRoot)) + groupsByRoot.set(groupRoot, []); + groupsByRoot.get(groupRoot).push(video); + } + + const duplicateGroups = Array.from(groupsByRoot.values()) + .filter(group => group.length > 1) + .map(group => group.sort((left, right) => left.file.localeCompare(right.file))) + .sort((left, right) => left[0].file.localeCompare(right[0].file)); + + if(mode === "print") { + for(const group of duplicateGroups) { + const keep = bestKeep(group, edges); + for(const video of group) { + if(video !== keep) + console.log(video.file); + } + } + } else { + const reviewRoot = path.resolve(root, out); + fs.mkdirSync(reviewRoot, { recursive: true }); + for(const [index, group] of duplicateGroups.entries()) { + const keep = bestKeep(group, edges); + const keepStem = path.parse(keep.file).name.slice(0, 48); + const groupDir = path.join(reviewRoot, `${String(index + 1).padStart(4, "0")}-${keepStem}`); + fs.mkdirSync(groupDir, { recursive: true }); + const seen = new Set(); + for(const video of [...group].sort((left, right) => Number(left.file !== keep.file) - Number(right.file !== keep.file))) { + const name = uniqueLinkName(groupDir, video.file, seen); + const link = path.join(groupDir, name); + if(fs.existsSync(link) || fs.lstatSync(link, { throwIfNoEntry: false })?.isSymbolicLink()) + fs.unlinkSync(link); + fs.symlinkSync(relativeLinkTarget(link, video.file), link); + } + } + } +} finally { + fs.rmSync(tmp, { recursive: true, force: true }); +} +js + +failed=0 +nix run 'nixpkgs#bun' -- $js $mode $root $out $threshold $high_threshold $duration_slop || failed=$status +cleanup +exit $failed