From 8dfc8bda2a208dd64de8b607ef90ecd66127a235 Mon Sep 17 00:00:00 2001 From: Kieran Klukas Date: Fri, 19 Dec 2025 18:10:50 +0700 Subject: [PATCH] feat: add cdn mover --- bun.lock | 63 +++++++++ package.json | 5 +- scripts/preprocess.ts | 14 +- scripts/rehost.ts | 320 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 395 insertions(+), 7 deletions(-) create mode 100755 scripts/rehost.ts diff --git a/bun.lock b/bun.lock index 1b4c0ff..c11f12a 100644 --- a/bun.lock +++ b/bun.lock @@ -4,8 +4,11 @@ "workspaces": { "": { "dependencies": { + "@types/cli-progress": "^3.11.6", + "cli-progress": "^3.12.0", "dotenv": "^16.4.7", "glob": "^13.0.0", + "sharp": "^0.34.5", }, "devDependencies": { "@types/bun": "latest", @@ -18,6 +21,58 @@ "@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.28.5", "", {}, "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q=="], + "@emnapi/runtime": ["@emnapi/runtime@1.7.1", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-PVtJr5CmLwYAU9PZDMITZoR5iAOShYREoR45EyyLrbntV50mdePTgUn4AmOw90Ifcj+x2kRjdzr1HP3RrNiHGA=="], + + "@img/colour": ["@img/colour@1.0.0", "", {}, "sha512-A5P/LfWGFSl6nsckYtjw9da+19jB8hkJ6ACTGcDfEJ0aE+l2n2El7dsVM7UVHZQ9s2lmYMWlrS21YLy2IR1LUw=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@isaacs/balanced-match": ["@isaacs/balanced-match@4.0.1", "", {}, "sha512-yzMTt9lEb8Gv7zRioUilSglI0c0smZ9k5D65677DLWLtWJaXIS3CqcGyUFByYKlnUj6TkjLVs54fBl6+TiGQDQ=="], "@isaacs/brace-expansion": ["@isaacs/brace-expansion@5.0.0", "", { "dependencies": { "@isaacs/balanced-match": "^4.0.1" } }, "sha512-ZT55BDLV0yv0RBm2czMiZ+SqCGO7AvmOM3G/w2xhVPH+te0aKgFjmBvGlL1dH+ql2tgGO3MVrbb3jCKyvpgnxA=="], @@ -28,6 +83,8 @@ "@types/bun": ["@types/bun@1.3.4", "", { "dependencies": { "bun-types": "1.3.4" } }, "sha512-EEPTKXHP+zKGPkhRLv+HI0UEX8/o+65hqARxLy8Ov5rIxMBPNTjeZww00CIihrIQGEQBYg+0roO5qOnS/7boGA=="], + "@types/cli-progress": ["@types/cli-progress@3.11.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-cE3+jb9WRlu+uOSAugewNpITJDt1VF8dHOopPO4IABFc3SXYL5WE/+PTz/FCdZRRfIujiWW3n3aMbv1eIGVRWA=="], + "@types/node": ["@types/node@24.10.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-WOhQTZ4G8xZ1tjJTvKOpyEVSGgOTvJAfDK3FNFgELyaTpzhdgHVHeqW8V+UJvzF5BT+/B54T/1S2K6gd9c7bbA=="], "@types/yauzl": ["@types/yauzl@2.10.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q=="], @@ -70,6 +127,8 @@ "chromium-bidi": ["chromium-bidi@0.11.0", "", { "dependencies": { "mitt": "3.0.1", "zod": "3.23.8" }, "peerDependencies": { "devtools-protocol": "*" } }, "sha512-6CJWHkNRoyZyjV9Rwv2lYONZf1Xm0IuDyNq97nwSsxxP3wf5Bwy15K5rOvVKMtJ127jJBmxFUanSAOjgFRxgrA=="], + "cli-progress": ["cli-progress@3.12.0", "", { "dependencies": { "string-width": "^4.2.3" } }, "sha512-tRkV3HJ1ASwm19THiiLIXLO7Im7wlTuKnvkYaTkyoAPefqjNg7W7DHKUlGRxy9vxDvbyCYQkQozvptuMkGCg8A=="], + "cliui": ["cliui@8.0.1", "", { "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", "wrap-ansi": "^7.0.0" } }, "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ=="], "color-convert": ["color-convert@2.0.1", "", { "dependencies": { "color-name": "~1.1.4" } }, "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ=="], @@ -84,6 +143,8 @@ "degenerator": ["degenerator@5.0.1", "", { "dependencies": { "ast-types": "^0.13.4", "escodegen": "^2.1.0", "esprima": "^4.0.1" } }, "sha512-TllpMR/t0M5sqCXfj85i4XaAzxmS5tVA16dqvdkMwGmzI+dXLXnw3J+3Vdv7VKw+ThlTMboK6i9rnZ6Nntj5CQ=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "devtools-protocol": ["devtools-protocol@0.0.1367902", "", {}, "sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg=="], "dotenv": ["dotenv@16.6.1", "", {}, "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow=="], @@ -190,6 +251,8 @@ "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="], "socks": ["socks@2.8.7", "", { "dependencies": { "ip-address": "^10.0.1", "smart-buffer": "^4.2.0" } }, "sha512-HLpt+uLy/pxB+bum/9DzAgiKS8CX1EvbWxI4zlmgGCExImLdiad2iCwXT5Z4c9c3Eq8rP2318mPW2c+QbtjK8A=="], diff --git a/package.json b/package.json index 7363bcc..3ca498b 100644 --- a/package.json +++ b/package.json @@ -9,8 +9,11 @@ "gen-og": "bun run scripts/genOG.ts" }, "dependencies": { + "@types/cli-progress": "^3.11.6", + "cli-progress": "^3.12.0", "dotenv": "^16.4.7", - "glob": "^13.0.0" + "glob": "^13.0.0", + "sharp": "^0.34.5" }, "devDependencies": { "@types/bun": "latest", diff --git a/scripts/preprocess.ts b/scripts/preprocess.ts index 85bcd96..c9a2cea 100644 --- a/scripts/preprocess.ts +++ b/scripts/preprocess.ts @@ -73,10 +73,11 @@ function transformImages(content: string): string { params.push(`class="${classes.join(' ')}"`); } - const keyValueMatches = attrs.matchAll(/([a-zA-Z]+)=["']?([^"'\s}]+)["']?/g); - for (const [, key, value] of keyValueMatches) { + const keyValueMatches = attrs.matchAll(/([a-zA-Z]+)=(?:"([^"]*)"|'([^']*)'|([^\s}]+))/g); + for (const [, key, doubleQuoted, singleQuoted, unquoted] of keyValueMatches) { if (key !== 'class') { - params.push(`${key}="${value.replace(/["']/g, '')}"`); + const value = doubleQuoted || singleQuoted || unquoted; + params.push(`${key}="${value}"`); } } } @@ -101,10 +102,11 @@ function transformImages(content: string): string { params.push(`class="${classes.join(' ')}"`); } - const keyValueMatches = attrs.matchAll(/([a-zA-Z]+)=["']?([^"'\s}]+)["']?/g); - for (const [, key, value] of keyValueMatches) { + const keyValueMatches = attrs.matchAll(/([a-zA-Z]+)=(?:"([^"]*)"|'([^']*)'|([^\s}]+))/g); + for (const [, key, doubleQuoted, singleQuoted, unquoted] of keyValueMatches) { if (key !== 'class') { - params.push(`${key}="${value.replace(/["']/g, '')}"`); + const value = doubleQuoted || singleQuoted || unquoted; + params.push(`${key}="${value}"`); } } } diff --git a/scripts/rehost.ts b/scripts/rehost.ts new file mode 100755 index 0000000..49442a9 --- /dev/null +++ b/scripts/rehost.ts @@ -0,0 +1,320 @@ +#!/usr/bin/env bun + +import fs from "fs"; +import path from "path"; +import { glob } from "glob"; +import cliProgress from "cli-progress"; +import sharp from "sharp"; + +const UPLOAD_URL = "https://l4.dunkirk.sh/upload"; +const AUTH_TOKEN = "crumpets"; +const contentDir = process.argv[2] || "content"; +const CONCURRENCY = 15; // Number of parallel uploads +const MAX_DIMENSION = 1920; // Max dimension for either width or height + +interface ImageMatch { + filePath: string; + originalUrl: string; + line: number; +} + +interface UploadResult { + url: string; + newUrl: string | null; + error?: string; +} + +async function uploadImage( + url: string, + progressBar?: cliProgress.SingleBar, +): Promise<{ newUrl: string | null; error?: string }> { + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(30000), // 30 second timeout + }); + + if (!response.ok) { + progressBar?.increment(); + return { + newUrl: null, + error: `Download failed: ${response.status} ${response.statusText}`, + }; + } + + const blob = await response.blob(); + let buffer = Buffer.from(await blob.arrayBuffer()); + + // Get file extension from URL or content type + const urlExt = url.split(".").pop()?.split("?")[0]; + const contentType = response.headers.get("content-type") || ""; + let ext = urlExt || "jpg"; + + if (contentType.includes("png")) ext = "png"; + else if (contentType.includes("jpeg") || contentType.includes("jpg")) + ext = "jpg"; + else if (contentType.includes("gif")) ext = "gif"; + else if (contentType.includes("webp")) ext = "webp"; + + // Resize image if it's too large + try { + const image = sharp(buffer); + const metadata = await image.metadata(); + + if (metadata.width && metadata.height) { + const maxDimension = Math.max(metadata.width, metadata.height); + + if (maxDimension > MAX_DIMENSION) { + // Resize so the longest side is MAX_DIMENSION + const isLandscape = metadata.width > metadata.height; + buffer = await image + .resize( + isLandscape ? MAX_DIMENSION : undefined, + isLandscape ? undefined : MAX_DIMENSION, + { + fit: 'inside', + withoutEnlargement: true, + } + ) + .toBuffer(); + } + } + } catch (resizeError) { + // If resize fails, continue with original buffer + console.error(`\nWarning: Failed to resize ${url}:`, resizeError); + } + + const filename = `image_${Date.now()}_${Math.random().toString(36).slice(2)}.${ext}`; + + // Create form data + const formData = new FormData(); + const file = new File([buffer], filename, { type: contentType }); + formData.append("file", file); + + const uploadResponse = await fetch(UPLOAD_URL, { + method: "POST", + headers: { + Authorization: `Bearer ${AUTH_TOKEN}`, + }, + body: formData, + signal: AbortSignal.timeout(30000), + }); + + if (!uploadResponse.ok) { + const errorText = await uploadResponse.text(); + progressBar?.increment(); + return { + newUrl: null, + error: `Upload failed: ${uploadResponse.status} ${uploadResponse.statusText} - ${errorText}`, + }; + } + + const result = await uploadResponse.json(); + progressBar?.increment(); + return { newUrl: result.url }; + } catch (error) { + progressBar?.increment(); + return { + newUrl: null, + error: `Exception: ${error instanceof Error ? error.message : String(error)}`, + }; + } +} + +async function processInBatches( + items: T[], + batchSize: number, + processor: (item: T) => Promise, +): Promise { + const results: R[] = []; + + for (let i = 0; i < items.length; i += batchSize) { + const batch = items.slice(i, i + batchSize); + const batchResults = await Promise.all(batch.map(processor)); + results.push(...batchResults); + } + + return results; +} + +function findImages(filePath: string): ImageMatch[] { + const content = fs.readFileSync(filePath, "utf8"); + const lines = content.split("\n"); + const images: ImageMatch[] = []; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + + // Find all image URLs in standard markdown: ![alt](url) or ![alt](url){attrs} + const singleImageRegex = /!\[([^\]]*)\]\(([^)]+)\)(?:\{[^}]+\})?/g; + let match; + + while ((match = singleImageRegex.exec(line)) !== null) { + const url = match[2]; + // Only process hel1 cdn URLs, skip gifs + if ( + url.includes("hc-cdn.hel1.your-objectstorage.com") && + !url.toLowerCase().endsWith(".gif") + ) { + images.push({ + filePath, + originalUrl: url, + line: i + 1, + }); + } + } + + // Find all image URLs in multi-image format: !![alt1](url1)[alt2](url2){attrs} + const multiImageRegex = /!!(\[([^\]]*)\]\(([^)]+)\))+(?:\{[^}]+\})?/g; + while ((match = multiImageRegex.exec(line)) !== null) { + const urlMatches = [...match[0].matchAll(/\[([^\]]*)\]\(([^)]+)\)/g)]; + for (const urlMatch of urlMatches) { + const url = urlMatch[2]; + // Only process hel1 cdn URLs, skip gifs + if ( + url.includes("hc-cdn.hel1.your-objectstorage.com") && + !url.toLowerCase().endsWith(".gif") + ) { + images.push({ + filePath, + originalUrl: url, + line: i + 1, + }); + } + } + } + } + + return images; +} + +function replaceImageUrl( + filePath: string, + oldUrl: string, + newUrl: string, +): void { + let content = fs.readFileSync(filePath, "utf8"); + + // Replace all occurrences of the old URL with the new one + // This handles both single and multi-image formats + content = content.replaceAll(oldUrl, newUrl); + + fs.writeFileSync(filePath, content); +} + +async function main() { + const files = glob.sync(`${contentDir}/**/*.md`); + const allImages: ImageMatch[] = []; + + // Find all images across all files + for (const file of files) { + const images = findImages(file); + allImages.push(...images); + } + + if (allImages.length === 0) { + console.log("No external images found to rehost."); + return; + } + + const uniqueUrls = [...new Set(allImages.map((img) => img.originalUrl))]; + console.log( + `Found ${uniqueUrls.length} unique images to rehost (${allImages.length} total references)\n`, + ); + + // Create progress bar + const progressBar = new cliProgress.SingleBar({ + format: + "Uploading |{bar}| {percentage}% | {value}/{total} images | ETA: {eta}s", + barCompleteChar: "\u2588", + barIncompleteChar: "\u2591", + hideCursor: true, + }); + + progressBar.start(uniqueUrls.length, 0); + + // Process URLs in parallel with concurrency limit + const urlMap = new Map(); + const failedUploads: { url: string; error: string }[] = []; + + const results = await processInBatches( + uniqueUrls, + CONCURRENCY, + async (url) => { + const result = await uploadImage(url, progressBar); + return { url, ...result }; + }, + ); + + progressBar.stop(); + + // Build URL map and collect errors + for (const { url, newUrl, error } of results) { + if (newUrl) { + urlMap.set(url, newUrl); + } else if (error) { + failedUploads.push({ url, error }); + } + } + + const successCount = urlMap.size; + const failCount = uniqueUrls.length - successCount; + + if (failCount > 0) { + console.log( + `\n⚠️ Failed to upload ${failCount} image${failCount === 1 ? "" : "s"}`, + ); + + // Group errors by type + const errorGroups = new Map(); + for (const { url, error } of failedUploads) { + const errorType = error.split(":")[0]; + if (!errorGroups.has(errorType)) { + errorGroups.set(errorType, []); + } + errorGroups.get(errorType)!.push(url); + } + + console.log("\nError summary:"); + for (const [errorType, urls] of errorGroups.entries()) { + console.log( + ` ${errorType}: ${urls.length} image${urls.length === 1 ? "" : "s"}`, + ); + if (urls.length <= 3) { + urls.forEach((url) => console.log(` - ${url}`)); + } else { + urls.slice(0, 2).forEach((url) => console.log(` - ${url}`)); + console.log(` ... and ${urls.length - 2} more`); + } + } + } + + if (urlMap.size === 0) { + console.log("\n❌ No images were successfully uploaded."); + return; + } + + // Replace URLs in files + console.log( + `\n✓ Successfully uploaded ${successCount} image${successCount === 1 ? "" : "s"}`, + ); + console.log("\nUpdating markdown files..."); + + let filesUpdated = 0; + const updatedFiles = new Set(); + + for (const [oldUrl, newUrl] of urlMap.entries()) { + const affectedImages = allImages.filter( + (img) => img.originalUrl === oldUrl, + ); + for (const img of affectedImages) { + replaceImageUrl(img.filePath, oldUrl, newUrl); + updatedFiles.add(img.filePath); + } + } + + console.log( + `✓ Updated ${updatedFiles.size} file${updatedFiles.size === 1 ? "" : "s"}\n`, + ); +} + +main(); -- 2.51.2