From ba93ad6df1e63e0cea2c8bca4357b2671f709ca9 Mon Sep 17 00:00:00 2001 From: Essem Date: Tue, 2 Dec 2025 21:54:16 -0600 Subject: [PATCH] refactor: Make room for more potential media types --- .env.example | 2 +- commands/general/raw.js | 2 +- commands/message/select-image.js | 18 ++-- config/servers.json | 2 +- docs/config.md | 4 +- natives/node/media.cc | 69 +++++++++++---- src/api/IMPLEMENTATION.md | 2 +- src/api/index.ts | 21 +++-- src/app.ts | 2 +- src/classes/mediaCommand.ts | 11 ++- src/utils/media.ts | 98 ++++++++++++--------- src/utils/mediaConnection.ts | 36 +++----- src/utils/mediaLib.ts | 11 ++- src/utils/mediaRunner.ts | 18 +++- src/utils/mediadetect.ts | 147 ++++++++++++++++--------------- src/utils/mime.ts | 16 ++++ src/utils/tempimages.ts | 2 +- src/utils/types.ts | 13 ++- 18 files changed, 288 insertions(+), 186 deletions(-) create mode 100644 src/utils/mime.ts diff --git a/.env.example b/.env.example index fdc9bc6e..6d53bb08 100644 --- a/.env.example +++ b/.env.example @@ -54,7 +54,7 @@ METRICS= # The media API type to be used # Set this to `none` to process all media locally -# Set this to `ws` if you want to use the external media API script, located in api/index.js +# Set this to `ws` if you want to use the external media API script, located in dist/api/index.js API_TYPE=none # Put ID of server to limit owner-only commands to diff --git a/commands/general/raw.js b/commands/general/raw.js index be6c761b..225c4beb 100644 --- a/commands/general/raw.js +++ b/commands/general/raw.js @@ -5,7 +5,7 @@ import imageDetect from "#utils/mediadetect.js"; class RawCommand extends Command { async run() { await this.acknowledge(); - const image = await imageDetect(this.client, this.permissions, this.message, this.interaction); + const image = await imageDetect(this.client, this.permissions, ["image"], this.message, this.interaction); if (image === undefined) { this.success = false; return this.getString("commands.responses.raw.noInput"); diff --git a/commands/message/select-image.js b/commands/message/select-image.js index 6ea08788..b2f65650 100644 --- a/commands/message/select-image.js +++ b/commands/message/select-image.js @@ -7,12 +7,18 @@ class SelectImageCommand extends Command { async run() { const message = this.interaction?.data.target; if (!(message instanceof Message)) throw Error("Target is not a message"); - const image = await imageDetect(this.client, this.permissions, message, this.interaction, true, false, true).catch( - (e) => { - if (e.name === "AbortError") return this.getString("image.timeout"); - throw e; - }, - ); + const image = await imageDetect( + this.client, + this.permissions, + ["image"], + message, + this.interaction, + true, + true, + ).catch((e) => { + if (e.name === "AbortError") return this.getString("image.timeout"); + throw e; + }); if (typeof image === "string") return image; this.success = false; if (image === undefined) { diff --git a/config/servers.json b/config/servers.json index 895bac77..57496f5a 100644 --- a/config/servers.json +++ b/config/servers.json @@ -1,5 +1,5 @@ { "lava": [{ "name": "localhost", "url": "localhost:2333", "auth": "youshallnotpass" }], - "image": [{ "name": "localhost", "server": "localhost", "auth": "verycoolpass100", "tls": false }], + "media": [{ "name": "localhost", "server": "localhost", "auth": "verycoolpass100", "tls": false }], "searx": ["https://searx.projectlounge.pw"] } diff --git a/docs/config.md b/docs/config.md index 99fadc37..699d2bc8 100644 --- a/docs/config.md +++ b/docs/config.md @@ -28,7 +28,7 @@ These variables that are not necessarily required for the bot to run, but can gr - `TMP_DOMAIN`: The root domain/directory that the media files larger than Discord's filesize limit are stored at. Example: `https://projectlounge.pw/tmp` - `THRESHOLD`: A filesize threshold that the bot will start deleting old files in `TEMPDIR` at. - `METRICS`: The HTTP port to serve [Prometheus](https://prometheus.io/)-compatible metrics on. -- `API_TYPE`: Set this to "none" if you want to process all media locally. Alternatively, set it to "ws" to use a media API server specified in the `image` block of `config/servers.json`. +- `API_TYPE`: Set this to "none" if you want to process all media locally. Alternatively, set it to "ws" to use a media API server specified in the `media` block of `config/servers.json`. - `ADMIN_SERVER`: A Discord server/guild ID to limit owner-only commands such as eval to. ## JSON @@ -73,7 +73,7 @@ The JSON-based configuration files are located in `config/`. "auth": "youshallnotpass" // Password/authorization code for the server } ], - "image": [ // Objects containing info for connecting to WS media server(s) + "media": [ // Objects containing info for connecting to WS media server(s) { "name": "test", // A human-friendly name for the server "server": "localhost", // IP address or domain name for the server diff --git a/natives/node/media.cc b/natives/node/media.cc index 3fe9cab8..719df511 100644 --- a/natives/node/media.cc +++ b/natives/node/media.cc @@ -23,15 +23,28 @@ namespace backward { using namespace std; +string getDefaultType(string cmdType) { + if (cmdType == "image") { + return "png"; + } + return ""; +} + Napi::Value ProcessMedia(const Napi::CallbackInfo &info) { Napi::Env env = info.Env(); - string command = info[0].As().Utf8Value(); - Napi::Object obj = info[1].As(); - Napi::Object input = info[2].As(); - string type = input.Has("type") ? input.Get("type").As().Utf8Value() : "png"; + string cmdType = info[0].As().Utf8Value(); + string command = info[1].As().Utf8Value(); + Napi::Object obj = info[2].As(); + Napi::Object input = info[3].As(); Napi::Promise::Deferred deferred = Napi::Promise::Deferred::New(env); + string type = input.Has("type") ? input.Get("type").As().Utf8Value() : getDefaultType(cmdType); + if (type == "") { + deferred.Reject(Napi::Error::New(env, "Job type \"" + cmdType + "\" is unknown").Value()); + return deferred.Promise(); + } + esmb::ArgumentMap Arguments; // We only have a single possible global arg at the moment, @@ -70,8 +83,14 @@ Napi::Value ProcessMedia(const Napi::CallbackInfo &info) { bufSize = data.ByteLength(); } - ImageAsyncWorker *asyncWorker = new ImageAsyncWorker(env, deferred, command, Arguments, type, bufData, bufSize); - asyncWorker->Queue(); + AsyncWorker *asyncWorker; + if (cmdType == "image") { + asyncWorker = new ImageAsyncWorker(env, deferred, command, Arguments, type, bufData, bufSize); + asyncWorker->Queue(); + return deferred.Promise(); + } + + deferred.Reject(Napi::Error::New(env, "Job type \"" + cmdType + "\" is unknown").Value()); return deferred.Promise(); } @@ -95,19 +114,23 @@ Napi::Value Trim(const Napi::CallbackInfo &info) { #endif } -void *checkTypes(GType type, Napi::Object *formats) { +void *checkTypes(GType type, Napi::Array *formats) { + Napi::Array arr = *formats; VipsObjectClass *c = VIPS_OBJECT_CLASS(g_type_class_ref(type)); + size_t i = 0; - if (strcmp(c->nickname, "jpegload")) formats->Set("image/jpeg", true); - if (strcmp(c->nickname, "pngload")) formats->Set("image/png", true); - if (strcmp(c->nickname, "gifload")) formats->Set("image/gif", true); - if (strcmp(c->nickname, "webpload")) formats->Set("image/webp", true); - if (strcmp(c->nickname, "heifload")) formats->Set("image/avif", true); + if (strcmp(c->nickname, "jpegload")) arr[i++] = Napi::String::From(formats->Env(), "image/jpeg"); + if (strcmp(c->nickname, "pngload")) arr[i++] = Napi::String::From(formats->Env(), "image/png"); + if (strcmp(c->nickname, "gifload")) arr[i++] = Napi::String::From(formats->Env(), "image/gif"); + if (strcmp(c->nickname, "webpload")) arr[i++] = Napi::String::From(formats->Env(), "image/webp"); + if (strcmp(c->nickname, "heifload")) arr[i++] = Napi::String::From(formats->Env(), "image/avif"); return NULL; } Napi::Value MediaInit(const Napi::CallbackInfo &info) { + Napi::Object formats = Napi::Object::New(info.Env()); + #if __GLIBC__ /* Set mmap threshold to 128kb to work around a similar glibc bug to the one above. @@ -119,11 +142,14 @@ Napi::Value MediaInit(const Napi::CallbackInfo &info) { */ mallopt(M_MMAP_THRESHOLD, 131072); #endif + #if defined(WIN32) && defined(MAGICK_ENABLED) Magick::InitializeMagick(""); #endif + if (VIPS_INIT("")) vips_error_exit(NULL); vips_cache_set_max(0); + #if VIPS_MAJOR_VERSION >= 8 && VIPS_MINOR_VERSION >= 13 vips_block_untrusted_set(true); vips_operation_block_set("VipsForeignLoad", true); @@ -133,30 +159,35 @@ Napi::Value MediaInit(const Napi::CallbackInfo &info) { vips_operation_block_set("VipsForeignLoadWebp", false); vips_operation_block_set("VipsForeignLoadHeif", false); #endif - Napi::Object formats = Napi::Object::New(info.Env()); - vips_type_map_all(g_type_from_name("VipsForeignLoad"), (VipsTypeMapFn)checkTypes, &formats); + + Napi::Array imgFormats = Napi::Array::New(info.Env()); + vips_type_map_all(g_type_from_name("VipsForeignLoad"), (VipsTypeMapFn)checkTypes, &imgFormats); + formats.Set("image", imgFormats); return formats; } Napi::Object Init(Napi::Env env, Napi::Object exports) { - exports.Set(Napi::String::New(env, "media"), Napi::Function::New(env, ProcessMedia)); + exports.Set(Napi::String::New(env, "process"), Napi::Function::New(env, ProcessMedia)); exports.Set(Napi::String::New(env, "init"), Napi::Function::New(env, MediaInit)); exports.Set(Napi::String::New(env, "trim"), Napi::Function::New(env, Trim)); - Napi::Array arr = Napi::Array::New(env); + Napi::Object funcs = Napi::Object::New(env); size_t i = 0; + + Napi::Array imageFuncs = Napi::Array::New(env); for (auto const &imap : esmb::Image::FunctionMap) { Napi::HandleScope scope(env); - arr[i] = Napi::String::New(env, imap.first); + imageFuncs[i] = Napi::String::New(env, imap.first); i++; } for (auto const &imap : esmb::Image::NoInputFunctionMap) { Napi::HandleScope scope(env); - arr[i] = Napi::String::New(env, imap.first); + imageFuncs[i] = Napi::String::New(env, imap.first); i++; } + funcs.Set("image", imageFuncs); - exports.Set(Napi::String::New(env, "funcs"), arr); + exports.Set(Napi::String::New(env, "funcs"), funcs); return exports; } diff --git a/src/api/IMPLEMENTATION.md b/src/api/IMPLEMENTATION.md index 72a969b9..cd9cd418 100644 --- a/src/api/IMPLEMENTATION.md +++ b/src/api/IMPLEMENTATION.md @@ -4,7 +4,7 @@ The esmBot Media API is a combined HTTP and WebSocket API. The default port to a ## HTTP -### GET `/image/?id=` +### GET `/media/?id=` Get output data after job is finished running. The Content-Type header is properly set. diff --git a/src/api/index.ts b/src/api/index.ts index 7f59a205..7af377d4 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -9,9 +9,9 @@ import { WebSocketServer, type ErrorEvent } from "ws"; import logger from "#utils/logger.js"; import { media } from "#utils/mediaLib.js"; import run from "#utils/mediaRunner.js"; -import type { MediaParams } from "#utils/types.js"; +import type { MediaFormats, MediaParams } from "#utils/types.js"; -const formats = Object.keys(media.init()); +const formats = media.init(); const cacheTimeout = 15 * 60 * 1000; // jobs are deleted 15 minutes after completion if not fetched @@ -141,9 +141,12 @@ wss.on("connection", (ws, request) => { ws.binaryType = "nodebuffer"; const cur = Buffer.alloc(2); cur.writeUInt16LE(jobs.size); - const cmdFormats: { [cmd: string]: string[] } = {}; - for (const cmd of media.funcs) { - cmdFormats[cmd] = formats; + const cmdFormats: MediaFormats = {}; + if (media.funcs.image && formats.image) { + cmdFormats.image = {}; + for (const cmd of media.funcs.image) { + cmdFormats.image[cmd] = formats.image; + } } const init = Buffer.concat([ Buffer.from([Rinit]), @@ -225,7 +228,7 @@ httpServer.on("request", (req, res) => { return res.end("400 Bad Request"); } const reqUrl = new URL(req.url, `http://${req.headers.host}`); - if (reqUrl.pathname === "/image" && req.method === "GET") { + if (reqUrl.pathname === "/media" && req.method === "GET") { const param = reqUrl.searchParams.get("id"); if (!param) { res.statusCode = 400; @@ -425,6 +428,12 @@ async function runJob(job: MiniJob, ws: WSocket): Promise { log(`Job ${job.id} starting...`, job.num); const object = job.msg; + + // Any other job type is invalid + if (object.type !== "image") { + throw new TypeError("Unknown job type"); + } + // If the input has a path, it must also have a type if (object.path && !object.input?.type) { throw new TypeError("Unknown media type"); diff --git a/src/app.ts b/src/app.ts index 7698b63f..c21c066d 100644 --- a/src/app.ts +++ b/src/app.ts @@ -145,7 +145,7 @@ if (database) { await database.setup(); } if (process.env.API_TYPE === "ws") await reloadMediaConnections(); -else initMediaLib(); +else await initMediaLib(); const shardArray = process.env.SHARDS && process.env.pm_id diff --git a/src/classes/mediaCommand.ts b/src/classes/mediaCommand.ts index b27cfaee..78eacf24 100644 --- a/src/classes/mediaCommand.ts +++ b/src/classes/mediaCommand.ts @@ -61,7 +61,14 @@ class MediaCommand extends Command { } const image = selection ?? - (await mediaDetect(this.client, this.permissions, this.message, this.interaction, true).catch((e) => { + (await mediaDetect( + this.client, + this.permissions, + staticProps.supportedTypes, + this.message, + this.interaction, + true, + ).catch((e) => { if (e.name === "AbortError") { runningCommands.delete(this.author.id); return this.getString("image.timeout"); @@ -89,7 +96,7 @@ class MediaCommand extends Command { } mediaParams = { cmd: staticProps.command, - type: "image", + type: image.mediaType ?? "image", params: { togif: !!this.getOptionBoolean("togif"), }, diff --git a/src/utils/media.ts b/src/utils/media.ts index 184e56e4..1d9defc4 100644 --- a/src/utils/media.ts +++ b/src/utils/media.ts @@ -1,18 +1,16 @@ import { Buffer } from "node:buffer"; import { lookup } from "node:dns/promises"; import fs from "node:fs"; -import { createRequire } from "node:module"; import process from "node:process"; import { fileTypeFromStream } from "file-type"; import ipaddr from "ipaddr.js"; -import serversConfig from "#config/servers.json" with { type: "json" }; import logger from "./logger.ts"; import MediaConnection from "./mediaConnection.ts"; import { random } from "./misc.ts"; import type { MediaParams, MediaTypeData } from "./types.ts"; const run = process.env.API_TYPE === "ws" ? null : (await import("./mediaRunner.ts")).default; -let img: import("./mediaLib.ts").MediaLib | undefined; +let mediaLib: import("./mediaLib.ts").MediaLib | undefined; interface ServerConfig { name: string; @@ -21,31 +19,25 @@ interface ServerConfig { tls?: boolean; } -const formats = [ - "image/jpeg", - "image/png", - "image/webp", - "image/gif", - "video/mp4", - "video/webm", - "video/quicktime", - "image/avif", -]; +export const formats = { + image: ["image/jpeg", "image/png", "image/webp", "image/gif", "image/avif"], +}; export const connections = new Map(); -export let servers: ServerConfig[] = process.env.API_TYPE === "ws" ? serversConfig.image : []; +export let servers: ServerConfig[]; -export function initMediaLib() { - const nodeRequire = createRequire(import.meta.url); - const imgLib = nodeRequire( - `../../build/${process.env.DEBUG && process.env.DEBUG === "true" ? "Debug" : "Release"}/esmbmedia.node`, - ); - imgLib.init(); - img = imgLib; +export async function initMediaLib() { + const { media } = await import("./mediaLib.ts"); + media.init(); + mediaLib = media; } -export async function getType(image: URL, extraReturnTypes: boolean): Promise { +export async function getType( + media: URL, + extraReturnTypes: boolean, + typeMedia: MediaParams["type"][], +): Promise { try { - const remoteIP = await lookup(image.host); + const remoteIP = await lookup(media.host); const parsedIP = ipaddr.parse(remoteIP.address); if (parsedIP.range() !== "unicast") return; } catch (e) { @@ -54,30 +46,31 @@ export async function getType(image: URL, extraReturnTypes: boolean): Promise { controller.abort(); }, 3000); try { - const imageRequest = await fetch(image, { + const mediaRequest = await fetch(media, { signal: controller.signal, method: "HEAD", }); clearTimeout(timeout); - if (imageRequest.redirected) { - const redirectHost = new URL(imageRequest.url).host; + if (mediaRequest.redirected) { + const redirectHost = new URL(mediaRequest.url).host; const remoteIP = await lookup(redirectHost); const parsedIP = ipaddr.parse(remoteIP.address); if (parsedIP.range() !== "unicast") return; } - url = imageRequest.url; + url = mediaRequest.url; let size = 0; - if (imageRequest.headers.has("content-range")) { - const contentRange = imageRequest.headers.get("content-range"); + if (mediaRequest.headers.has("content-range")) { + const contentRange = mediaRequest.headers.get("content-range"); if (contentRange) size = Number.parseInt(contentRange.split("/")[1]); - } else if (imageRequest.headers.has("content-length")) { - const contentLength = imageRequest.headers.get("content-length"); + } else if (mediaRequest.headers.has("content-length")) { + const contentLength = mediaRequest.headers.get("content-length"); if (contentLength) size = Number.parseInt(contentLength); } if (size > 41943040 && extraReturnTypes) { @@ -85,9 +78,12 @@ export async function getType(image: URL, extraReturnTypes: boolean): Promise { controller.abort(); @@ -100,16 +96,27 @@ export async function getType(image: URL, extraReturnTypes: boolean): Promise formats[v]))].includes( + fileType.mime, + ) + ) + return; + + const typePrefix = fileType.mime.split("/")[0] as MediaParams["type"]; + if (!typeMedia.includes(typePrefix)) return; + mediaType = typePrefix; + + if (mediaType) type = fileType.mime; } } } } finally { clearTimeout(timeout); } - return { type, url }; + return { type, url, mediaType }; } function connect(server: string, auth: string | undefined, name: string | undefined, tls?: boolean) { @@ -126,7 +133,16 @@ export function disconnect() { async function repopulate() { const data = await fs.promises.readFile(new URL("../../config/servers.json", import.meta.url), { encoding: "utf8" }); - servers = JSON.parse(data).image; + const parsed = JSON.parse(data); + if (parsed.image) { + logger.warn('!!! THE "image" FIELD IN config/servers.json IS DEPRECATED !!!'); + logger.warn( + 'The "image" field has been renamed to "media". Please rename it in your config; esmBot will no longer read this field in a future version.', + ); + servers = parsed.image; + } else { + servers = parsed.media; + } } export async function reloadMediaConnections() { @@ -150,11 +166,11 @@ async function getIdeal(object: MediaParams): Promise { running--; if (running < 0) running = 0; - if (img && running === 0) { - img.trim(); + if (mediaLib && running === 0) { + mediaLib.trim(); } }); return { diff --git a/src/utils/mediaConnection.ts b/src/utils/mediaConnection.ts index 50246b4b..528a0ffb 100644 --- a/src/utils/mediaConnection.ts +++ b/src/utils/mediaConnection.ts @@ -2,6 +2,8 @@ import { Buffer } from "node:buffer"; import { setTimeout } from "node:timers/promises"; import WSocket, { type Data, type ErrorEvent } from "ws"; import logger from "./logger.ts"; +import { mimeToExt } from "./mime.ts"; +import type { MediaFormats, MediaFuncs } from "./types.ts"; const Rerror = 0x01; const Tqueue = 0x02; @@ -30,8 +32,8 @@ class MediaConnection { name?: string; tag: number; disconnected: boolean; - formats: { [key: string]: string[] }; - funcs: string[]; + formats: MediaFormats; + funcs: MediaFuncs; wsproto: string; sockurl: string; conn: WSocket; @@ -45,7 +47,7 @@ class MediaConnection { this.tag = 0; this.disconnected = false; this.formats = {}; - this.funcs = []; + this.funcs = {}; if (tls) { this.wsproto = "wss"; } else { @@ -76,7 +78,9 @@ class MediaConnection { logger.debug(`Received message from media server ${this.host} with opcode ${op}`); if (op === Rinit) { this.formats = JSON.parse(msg.toString("utf8", 7)); - this.funcs = Object.keys(this.formats); + this.funcs = { + image: this.formats.image ? Object.keys(this.formats.image) : [], + }; return; } if (op === Rclose) { @@ -163,7 +167,7 @@ class MediaConnection { async getOutput(jobid: string) { logger.debug(`Getting output of ${jobid} on media server ${this.host}`); const req = await fetch( - `${this.httpurl}/image?id=${jobid}`, + `${this.httpurl}/media?id=${jobid}`, this.auth ? { headers: { @@ -173,27 +177,7 @@ class MediaConnection { : undefined, ); const contentType = req.headers.get("content-type"); - let type: string; - switch (contentType) { - case "image/gif": - type = "gif"; - break; - case "image/png": - type = "png"; - break; - case "image/jpeg": - type = "jpg"; - break; - case "image/webp": - type = "webp"; - break; - case "image/avif": - type = "avif"; - break; - default: - type = contentType ?? "unknown"; - break; - } + const type = contentType ? mimeToExt(contentType) : "unknown"; return { buffer: Buffer.from(await req.arrayBuffer()), type }; } diff --git a/src/utils/mediaLib.ts b/src/utils/mediaLib.ts index f0528fce..47357baa 100644 --- a/src/utils/mediaLib.ts +++ b/src/utils/mediaLib.ts @@ -4,14 +4,19 @@ import process from "node:process"; import type { MediaParams } from "./types.ts"; export interface MediaLib { - funcs: string[]; + funcs: { + image: string[]; + }; - media( + process( + type: MediaParams["type"], cmd: string, params: MediaParams["params"], input: MediaParams["input"], ): Promise<{ data: Buffer; type: string }>; - init(): Record; + init(): { + image?: string[]; + }; trim(): number; } diff --git a/src/utils/mediaRunner.ts b/src/utils/mediaRunner.ts index 2bdf26e0..05499faa 100644 --- a/src/utils/mediaRunner.ts +++ b/src/utils/mediaRunner.ts @@ -2,11 +2,16 @@ import { Buffer } from "node:buffer"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { media } from "./mediaLib.ts"; +import { mimeToExt } from "./mime.ts"; import type { MediaParams } from "./types.ts"; +const defaultExts = { + image: "png", +}; + export default async function run(object: MediaParams): Promise<{ buffer: Buffer; fileExtension: string }> { // Check if command exists - if (!media.funcs.includes(object.cmd)) { + if (!media.funcs[object.type]?.includes(object.cmd)) { return { buffer: Buffer.alloc(0), fileExtension: "nocmd", @@ -16,7 +21,12 @@ export default async function run(object: MediaParams): Promise<{ buffer: Buffer let inputBuffer: ArrayBuffer | null = null; if (object.path) { // If the file has a path, it must also have a type - if (object.input?.type !== "image/gif" && object.input?.type !== "image/webp" && object.onlyAnim) { + if ( + object.type === "image" && + object.input?.type !== "image/gif" && + object.input?.type !== "image/webp" && + object.onlyAnim + ) { return { buffer: Buffer.alloc(0), fileExtension: "noanim", @@ -46,14 +56,14 @@ export default async function run(object: MediaParams): Promise<{ buffer: Buffer // Convert from a MIME type (e.g. "image/png") to something the media processor understands (e.g. "png"). // Don't set `type` directly on the object we are passed as it will be read afterwards. // If no type is given (say, the command generates its own output), make it a PNG. - const fileExtension = object.input?.type?.split("/")[1] ?? "png"; + const fileExtension = object.input?.type ? mimeToExt(object.input.type) : defaultExts[object.type]; if (object.input) { if (inputBuffer) object.input.data = inputBuffer; object.input.type = fileExtension; } object.params.basePath = path.join(path.dirname(fileURLToPath(import.meta.url)), "../../"); - const { data, type } = await media.media(object.cmd, object.params, object.input ?? {}); + const { data, type } = await media.process(object.type, object.cmd, object.params, object.input ?? {}); return { buffer: data, fileExtension: type, diff --git a/src/utils/mediadetect.ts b/src/utils/mediadetect.ts index d2e40abd..9fa2af28 100644 --- a/src/utils/mediadetect.ts +++ b/src/utils/mediadetect.ts @@ -12,8 +12,8 @@ import { ThreadChannel, } from "oceanic.js"; import logger from "./logger.ts"; -import { getType } from "./media.ts"; -import type { MediaTypeData } from "./types.ts"; +import { formats, getType } from "./media.ts"; +import type { MediaParams, MediaTypeData } from "./types.ts"; const tenorURLs = ["tenor.com", "www.tenor.com"]; const giphyURLs = ["giphy.com", "www.giphy.com", "i.giphy.com"]; @@ -31,9 +31,6 @@ const combined = [...tenorURLs, ...giphyURLs, ...giphyMediaURLs]; const providerUrls = ["https://tenor.co", "https://giphy.com"]; -const imageFormats = ["image/jpeg", "image/png", "image/webp", "image/gif", "image/avif", "large"]; -const videoFormats = ["video/mp4", "video/webm", "video/mov"]; - type TenorMediaObject = { url: string; dims: number[]; @@ -58,54 +55,55 @@ export type MediaMeta = { url: string; name: string; spoiler: boolean; + mediaType?: MediaParams["type"]; }; /** - * Gets proper image paths. + * Gets proper media paths. */ -async function getImage( - image: string, - image2: string, - video: boolean, +async function getMedia( + media: string, + media2: string, + mediaType: MediaParams["type"][], spoiler = false, extraReturnTypes = false, type: string | null = null, client: Client | undefined = undefined, ): Promise { - let imageURL: URL; + let mediaURL: URL; try { - imageURL = new URL(image); - if (!imageURL.host) throw null; - if (imageURL.protocol !== "http:" && imageURL.protocol !== "https:") throw null; + mediaURL = new URL(media); + if (!mediaURL.host) throw null; + if (mediaURL.protocol !== "http:" && mediaURL.protocol !== "https:") throw null; } catch { return { - url: image2, - path: image, + url: media2, + path: media, name: "null", type: "badurl", spoiler, }; } - const fileNameSplit = imageURL.pathname.split("/"); + const fileNameSplit = mediaURL.pathname.split("/"); const fileName = fileNameSplit[fileNameSplit.length - 1]; const fileNameNoExtension = fileName.slice(0, fileName.lastIndexOf(".")); const payload: MediaMeta = { - url: image2, - path: image, + url: media2, + path: media, name: fileNameNoExtension, spoiler, }; - const host = new URL(image2).host; - if (combined.includes(host)) { + const host = new URL(media2).host; + if (mediaType.includes("image") && combined.includes(host)) { if (tenorURLs.includes(host)) { // Tenor doesn't let us access a raw GIF without going through their API, // so we use that if there's a key in the config if (process.env.TENOR !== "") { let id: string | undefined; - if (image2.includes("tenor.com/view/")) { - id = image2.split("-").pop(); - } else if (image2.endsWith(".gif")) { - const redirect = (await fetch(image2, { method: "HEAD", redirect: "manual" })).headers.get("location"); + if (media2.includes("tenor.com/view/")) { + id = media2.split("-").pop(); + } else if (media2.endsWith(".gif")) { + const redirect = (await fetch(media2, { method: "HEAD", redirect: "manual" })).headers.get("location"); id = redirect?.split("-").pop(); } else { return; @@ -128,44 +126,50 @@ async function getImage( return; } payload.type = "image/gif"; + payload.mediaType = "image"; } else if (giphyURLs.includes(host)) { // Can result in an HTML page instead of a WEBP - payload.path = `https://media0.giphy.com/media/${image2.split("/")[4].split("-").pop()}/giphy.webp`; + payload.path = `https://media0.giphy.com/media/${media2.split("/")[4].split("-").pop()}/giphy.webp`; payload.type = "image/webp"; + payload.mediaType = "image"; } else if (giphyMediaURLs.includes(host)) { - payload.path = `https://media0.giphy.com/media/${image2.split("/")[4]}/giphy.webp`; + payload.path = `https://media0.giphy.com/media/${media2.split("/")[4]}/giphy.webp`; payload.type = "image/webp"; + payload.mediaType = "image"; } } else { let result: MediaTypeData | undefined; if ( - (imageURL.host === "cdn.discordapp.com" || imageURL.host === "media.discordapp.net") && - imageURL.pathname.match(/^\/(?:ephemeral-)?attachments\/\d+\/\d+\//) + (mediaURL.host === "cdn.discordapp.com" || mediaURL.host === "media.discordapp.net") && + mediaURL.pathname.match(/^\/(?:ephemeral-)?attachments\/\d+\/\d+\//) ) { let url: URL; - if (client && isAttachmentExpired(imageURL)) { - const refreshed = await client.rest.misc.refreshAttachmentURLs([image]); + if (client && isAttachmentExpired(mediaURL)) { + const refreshed = await client.rest.misc.refreshAttachmentURLs([media]); url = new URL(refreshed.refreshedURLs[0].refreshed); } else { - url = new URL(image); + url = new URL(media); } - url.searchParams.set("animated", "true"); - result = await getType(url, extraReturnTypes); + if (mediaType.includes("image")) url.searchParams.set("animated", "true"); + result = await getType(url, extraReturnTypes, mediaType); } else if ( - (imageURL.host === "images-ext-1.discordapp.net" || imageURL.host === "images-ext-2.discordapp.net") && - imageURL.pathname.match(/^\/external\/[\w-]+\//) + mediaType.includes("image") && + (mediaURL.host === "images-ext-1.discordapp.net" || mediaURL.host === "images-ext-2.discordapp.net") && + mediaURL.pathname.match(/^\/external\/[\w-]+\//) ) { - imageURL.searchParams.set("animated", "true"); - result = await getType(imageURL, extraReturnTypes); + mediaURL.searchParams.set("animated", "true"); + result = await getType(mediaURL, extraReturnTypes, mediaType); } else { - result = await getType(imageURL, extraReturnTypes); + result = await getType(mediaURL, extraReturnTypes, mediaType); } if (!result) return; if (result.url) payload.path = result.url; payload.type = type ?? result.type; + if (result.mediaType) payload.mediaType = result.mediaType; if ( !payload.type || - ((video ? !videoFormats.includes(payload.type) : true) && !imageFormats.includes(payload.type)) + !payload.mediaType || + ![...(mediaType.length === 0 ? formats.image : mediaType.flatMap((v) => formats[v]))].includes(payload.type) ) return; } @@ -173,28 +177,28 @@ async function getImage( } /** - * Checks a single message for videos or images + * Checks a single message for media */ -async function checkImages( +async function checkMedia( message: Message, extraReturnTypes: boolean, - video: boolean, + mediaType: MediaParams["type"][], ): Promise { let type: MediaMeta | undefined; // first check the embeds if (message.embeds.length !== 0) { - type = await checkEmbeds(message, extraReturnTypes, video); + type = await checkEmbeds(message, extraReturnTypes, mediaType); } // then check the attachments if (!type && message.attachments.size !== 0) { const firstAttachment = message.attachments.first(); - if (firstAttachment?.width) - type = await getImage( + if (firstAttachment) + type = await getMedia( firstAttachment.proxyURL, firstAttachment.url, - video, + mediaType, !!(firstAttachment.flags & AttachmentFlags.IS_SPOILER), ); } @@ -202,16 +206,15 @@ async function checkImages( // then check embeds and attachments inside forwards if (!type && message.messageSnapshots?.[0]) { const forward = message.messageSnapshots?.[0].message; - if (forward.embeds.length !== 0) type = await checkEmbeds(forward, extraReturnTypes, video); + if (forward.embeds.length !== 0) type = await checkEmbeds(forward, extraReturnTypes, mediaType); if (!type && forward.attachments.length !== 0) { - if (forward.attachments[0].width) - type = await getImage( - forward.attachments[0].proxyURL, - forward.attachments[0].url, - video, - !!(forward.attachments[0].flags & AttachmentFlags.IS_SPOILER), - ); + type = await getMedia( + forward.attachments[0].proxyURL, + forward.attachments[0].url, + mediaType, + !!(forward.attachments[0].flags & AttachmentFlags.IS_SPOILER), + ); } } @@ -219,7 +222,11 @@ async function checkImages( return type; } -function checkEmbeds(message: Message | MessageSnapshotMessage, extraReturnTypes: boolean, video: boolean) { +function checkEmbeds( + message: Message | MessageSnapshotMessage, + extraReturnTypes: boolean, + mediaType: MediaParams["type"][], +) { let hasSpoiler = false; if (message.embeds[0].url && message.content) { const spoilerRegex = /\|\|.*https?:\/\/.*\|\|/s; @@ -232,22 +239,22 @@ function checkEmbeds(message: Message | MessageSnapshotMessage, extraReturnTypes message.embeds[0].video?.url && message.embeds[0].url ) { - return getImage(message.embeds[0].video.url, message.embeds[0].url, video, hasSpoiler, extraReturnTypes); + return getMedia(message.embeds[0].video.url, message.embeds[0].url, mediaType, hasSpoiler, extraReturnTypes); // then thumbnails } else if (message.embeds[0].thumbnail) { - return getImage( + return getMedia( message.embeds[0].thumbnail.proxyURL ?? message.embeds[0].thumbnail.url, message.embeds[0].thumbnail.url, - video, + mediaType, hasSpoiler, extraReturnTypes, ); // and finally direct images } else if (message.embeds[0].image) { - return getImage( + return getMedia( message.embeds[0].image.proxyURL ?? message.embeds[0].image.url, message.embeds[0].image.url, - video, + mediaType, hasSpoiler, extraReturnTypes, ); @@ -309,33 +316,33 @@ export async function stickerDetect( } /** - * Checks for the latest message containing an image and returns the URL of the image. + * Checks for the latest message containing media and returns the URL of said media. */ export default async ( client: Client, perms: Permission, + mediaType: MediaParams["type"][], cmdMessage?: Message, interaction?: CommandInteraction, extraReturnTypes = false, - video = false, singleMessage = false, ): Promise => { // we start by determining whether or not we're dealing with an interaction or a message if (interaction) { // we can get a raw attachment or a URL in the interaction itself - const attachment = interaction.data.options.getAttachment("image"); + const attachment = interaction.data.options.getAttachment(mediaType.length === 1 ? mediaType[0] : "media"); if (attachment) { - return getImage( + return getMedia( attachment.proxyURL, attachment.url, - video, + mediaType, !!(attachment.flags & AttachmentFlags.IS_SPOILER), !!attachment.contentType, ); } const link = interaction.data.options.getString("link"); if (link) { - return getImage(link, link, video, false, extraReturnTypes, null, interaction.client); + return getMedia(link, link, mediaType, false, extraReturnTypes, null, interaction.client); } } if (cmdMessage) { @@ -345,12 +352,12 @@ export default async ( .getMessage(cmdMessage.messageReference.channelID, cmdMessage.messageReference.messageID) .catch(() => undefined); if (replyMessage) { - const replyResult = await checkImages(replyMessage, extraReturnTypes, video); + const replyResult = await checkMedia(replyMessage, extraReturnTypes, mediaType); if (replyResult) return replyResult; } } // then we check the current message - const result = await checkImages(cmdMessage, extraReturnTypes, video); + const result = await checkMedia(cmdMessage, extraReturnTypes, mediaType); if (result) return result; } if (!singleMessage && (cmdMessage || interaction?.authorizingIntegrationOwners?.[0] !== undefined)) { @@ -372,7 +379,7 @@ export default async ( const messages = await channel.getMessages(); // iterate over each message for (const message of messages) { - const result = await checkImages(message, extraReturnTypes, video); + const result = await checkMedia(message, extraReturnTypes, mediaType); if (result) return result; } } diff --git a/src/utils/mime.ts b/src/utils/mime.ts new file mode 100644 index 00000000..2dbefe3b --- /dev/null +++ b/src/utils/mime.ts @@ -0,0 +1,16 @@ +export function mimeToExt(mime: string) { + switch (mime) { + case "image/gif": + return "gif"; + case "image/png": + return "png"; + case "image/jpeg": + return "jpg"; + case "image/webp": + return "webp"; + case "image/avif": + return "avif"; + default: + return mime.split("/")[1] ?? "unknown"; + } +} diff --git a/src/utils/tempimages.ts b/src/utils/tempimages.ts index d28f6d21..d345ae43 100644 --- a/src/utils/tempimages.ts +++ b/src/utils/tempimages.ts @@ -78,7 +78,7 @@ export async function upload( ); } if (save) { - const type = await getType(new URL(imageURL), true); + const type = await getType(new URL(imageURL), true, []); selectedImages.set(authorId, { url: imageURL, path: type?.url ?? imageURL, diff --git a/src/utils/types.ts b/src/utils/types.ts index ae392607..4c49c3a5 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -62,8 +62,8 @@ export interface CommandInfo { } export interface MediaParams { - type: "image"; cmd: string; + type: "image"; params: { [key: string]: string | number | boolean; }; @@ -84,6 +84,17 @@ export interface MediaParams { export interface MediaTypeData { url?: string; type?: string; + mediaType?: MediaParams["type"]; +} + +export interface MediaFormats { + image?: { + [cmd: string]: string[]; + }; +} + +export interface MediaFuncs { + image?: string[]; } export interface SearXNGResults { -- 2.51.2