From e15d3485bf35bc0f1869a12c36a1e79ee2178ba0 Mon Sep 17 00:00:00 2001 From: Essem Date: Mon, 27 Apr 2026 18:44:38 -0500 Subject: [PATCH] refactor: Media handling overhaul This heavily reworks media handling in various areas. More media checking has been moved from the bot code to the media processing code/server, a single request is now used to both check the media type and get the full buffer, some areas have been simplified, and a good amount of cleanup was performed overall. This also makes the text input in uncanny required, removes URL filtering on certain commands, and removes the URL/filename arguments from the params/criteria functions. As this info is now obtained during image processing, it is no longer accessible beforehand. --- commands/fun/homebrew.js | 7 +- commands/image-editing/caption.js | 7 +- commands/image-editing/caption2.js | 7 +- commands/image-editing/freeze.js | 1 - commands/image-editing/meme.js | 18 +- commands/image-editing/motivate.js | 18 +- commands/image-editing/reddit.js | 7 +- commands/image-editing/reverse.js | 1 - commands/image-editing/slow.js | 1 - commands/image-editing/snapchat.js | 7 +- commands/image-editing/soos.js | 1 - commands/image-editing/speed.js | 1 - commands/image-editing/spotify.js | 7 +- commands/image-editing/uncanny.js | 36 ++-- commands/image-editing/unfreeze.js | 1 - commands/image-editing/whisper.js | 7 +- commands/message/select-image.js | 40 ++--- locales/en-US.json | 3 - natives/common/maps.h | 8 + natives/node/media.cc | 12 +- src/api/IMPLEMENTATION.md | 23 ++- src/api/index.ts | 48 +++--- src/classes/mediaCommand.ts | 114 ++++--------- src/events/interactionCreate.ts | 3 - src/utils/collections.ts | 3 +- src/utils/media.ts | 178 ++++++++++++-------- src/utils/mediaConnection.ts | 18 +- src/utils/mediaLib.ts | 17 +- src/utils/mediaRunner.ts | 146 ++++++++++------ src/utils/mediadetect.ts | 256 +++++++++++------------------ src/utils/tempimages.ts | 7 +- src/utils/types.ts | 29 ++-- 32 files changed, 482 insertions(+), 550 deletions(-) diff --git a/commands/fun/homebrew.js b/commands/fun/homebrew.js index fe01ad72..54ec38cb 100644 --- a/commands/fun/homebrew.js +++ b/commands/fun/homebrew.js @@ -1,11 +1,8 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class HomebrewCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); return { caption: this.clean(newArgs), }; diff --git a/commands/image-editing/caption.js b/commands/image-editing/caption.js index 9604c6ab..a61d032b 100644 --- a/commands/image-editing/caption.js +++ b/commands/image-editing/caption.js @@ -1,11 +1,8 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class CaptionCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); let newCaption = this.clean(newArgs); const currentDate = new Date(); const isApril1 = currentDate.getDate() === 1 && currentDate.getMonth() === 3; diff --git a/commands/image-editing/caption2.js b/commands/image-editing/caption2.js index aa48e829..d9184506 100644 --- a/commands/image-editing/caption2.js +++ b/commands/image-editing/caption2.js @@ -25,11 +25,8 @@ const words = [ ]; class CaptionTwoCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); const font = this.getOptionString("font"); return { caption: newArgs?.trim() diff --git a/commands/image-editing/freeze.js b/commands/image-editing/freeze.js index 517c580f..d56178eb 100644 --- a/commands/image-editing/freeze.js +++ b/commands/image-editing/freeze.js @@ -24,7 +24,6 @@ class FreezeCommand extends MediaCommand { static description = "Makes an image sequence only play once"; static aliases = ["noloop", "once"]; - static requiresAnim = true; static alwaysGIF = true; static noImage = "You need to provide an image/GIF to freeze!"; static command = "freeze"; diff --git a/commands/image-editing/meme.js b/commands/image-editing/meme.js index e1aa5d6a..412bd647 100644 --- a/commands/image-editing/meme.js +++ b/commands/image-editing/meme.js @@ -3,27 +3,17 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class MemeCommand extends MediaCommand { /** * @param {string} text - * @param {string | undefined} url */ - async criteria(text, url) { + async criteria(text) { if (typeof text !== "string") return false; - const [topText, bottomText] = text - .replaceAll(url ?? "", "") - .split(/(? elem.trim()); + const [topText, bottomText] = text.split(/(? elem.trim()); if (topText === "" && (!bottomText || bottomText === "")) return false; return true; } - /** - * @param {string | undefined} url - */ - paramsFunc(url) { + paramsFunc() { const newArgs = this.getOptionString("text") ?? this.args.join(" "); - const [topText, bottomText] = newArgs - .replaceAll(url ?? "", "") - .split(/(? elem.trim()); + const [topText, bottomText] = newArgs.split(/(? elem.trim()); const font = this.getOptionString("font"); return { topText: this.clean(this.getOptionBoolean("case") ? topText : topText.toUpperCase()), diff --git a/commands/image-editing/motivate.js b/commands/image-editing/motivate.js index c13a57fc..f64865df 100644 --- a/commands/image-editing/motivate.js +++ b/commands/image-editing/motivate.js @@ -3,27 +3,17 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class MotivateCommand extends MediaCommand { /** * @param {string} text - * @param {string | undefined} url */ - async criteria(text, url) { + async criteria(text) { if (typeof text !== "string") return false; - const [topText, bottomText] = text - .replaceAll(url ?? "", "") - .split(/(? elem.trim()); + const [topText, bottomText] = text.split(/(? elem.trim()); if (topText === "" && (!bottomText || bottomText === "")) return false; return true; } - /** - * @param {string | undefined} url - */ - paramsFunc(url) { + paramsFunc() { const newArgs = this.getOptionString("text") ?? this.args.join(" "); - const [topText, bottomText] = newArgs - .replaceAll(url ?? "", "") - .split(/(? elem.trim()); + const [topText, bottomText] = newArgs.split(/(? elem.trim()); const font = this.getOptionString("font"); return { topText: this.clean(topText), diff --git a/commands/image-editing/reddit.js b/commands/image-editing/reddit.js index aacff6f0..72d8919c 100644 --- a/commands/image-editing/reddit.js +++ b/commands/image-editing/reddit.js @@ -3,11 +3,8 @@ import { random } from "#utils/misc.js"; const names = ["esmBot", "me_irl", "dankmemes", "hmmm", "gaming", "wholesome", "chonkers", "memes", "funny", "lies"]; class RedditCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); const cleaned = this.clean(newArgs); return { caption: cleaned.trim() ? cleaned.replaceAll("\n", "").replaceAll(" ", "") : random(names), diff --git a/commands/image-editing/reverse.js b/commands/image-editing/reverse.js index e536044a..2734ce91 100644 --- a/commands/image-editing/reverse.js +++ b/commands/image-editing/reverse.js @@ -4,7 +4,6 @@ class ReverseCommand extends MediaCommand { static description = "Reverses an image sequence"; static aliases = ["backwards"]; - static requiresAnim = true; static alwaysGIF = true; static noImage = "You need to provide an image/GIF to reverse!"; static command = "reverse"; diff --git a/commands/image-editing/slow.js b/commands/image-editing/slow.js index a4a8edf2..397d7f42 100644 --- a/commands/image-editing/slow.js +++ b/commands/image-editing/slow.js @@ -25,7 +25,6 @@ class SlowCommand extends MediaCommand { static description = "Makes an image sequence slower"; static aliases = ["slowdown", "slower", "gifspeed2"]; - static requiresAnim = true; static alwaysGIF = true; static noImage = "You need to provide an image/GIF to slow down!"; static command = "speed"; diff --git a/commands/image-editing/snapchat.js b/commands/image-editing/snapchat.js index 4d12efd7..65e28c8d 100644 --- a/commands/image-editing/snapchat.js +++ b/commands/image-editing/snapchat.js @@ -1,11 +1,8 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class SnapchatCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); const position = this.getOptionNumber("position"); return { caption: this.clean(newArgs), diff --git a/commands/image-editing/soos.js b/commands/image-editing/soos.js index 80cbe99d..276cbe5d 100644 --- a/commands/image-editing/soos.js +++ b/commands/image-editing/soos.js @@ -8,7 +8,6 @@ class SooSCommand extends MediaCommand { static description = '"Loops" an image sequence by reversing it when it\'s finished'; static aliases = ["boomerang"]; - static requiresAnim = true; static alwaysGIF = true; static noImage = "You need to provide an image/GIF to loop!"; static command = "reverse"; diff --git a/commands/image-editing/speed.js b/commands/image-editing/speed.js index 55ab59c3..241350d6 100644 --- a/commands/image-editing/speed.js +++ b/commands/image-editing/speed.js @@ -24,7 +24,6 @@ class SpeedCommand extends MediaCommand { static description = "Makes an image sequence faster"; static aliases = ["speedup", "fast", "gifspeed", "faster"]; - static requiresAnim = true; static alwaysGIF = true; static noImage = "You need to provide an image/GIF to speed up!"; static command = "speed"; diff --git a/commands/image-editing/spotify.js b/commands/image-editing/spotify.js index cdc8ca86..68598aa9 100644 --- a/commands/image-editing/spotify.js +++ b/commands/image-editing/spotify.js @@ -1,11 +1,8 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class SpotifyCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); return { caption: this.clean(newArgs), }; diff --git a/commands/image-editing/uncanny.js b/commands/image-editing/uncanny.js index 40ea4d2f..5f51665b 100644 --- a/commands/image-editing/uncanny.js +++ b/commands/image-editing/uncanny.js @@ -3,20 +3,7 @@ import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import MediaCommand from "#cmd-classes/mediaCommand.js"; import { random } from "#utils/misc.js"; -const prompts = [ - "you found:", - "your dad is:", - "you ate:", - "your mom is:", - "your sister is:", - "you saw:", - "you get lost in:", - "you find:", - "you grab:", - "you pull out of your pocket:", - "you fight:", - "it's in your room:", -]; + const names = readdirSync(resolve(dirname(fileURLToPath(import.meta.url)), "../../assets/images/uncanny/")) .filter((val) => { if (!val.startsWith(".") && val.endsWith(".png")) return true; @@ -27,19 +14,22 @@ const names = readdirSync(resolve(dirname(fileURLToPath(import.meta.url)), "../. class UncannyCommand extends MediaCommand { /** - * @param {string | undefined} url + * @param {string} text */ - paramsFunc(url, name = "unknown") { + async criteria(text) { + if (typeof text !== "string") return false; + const [text1, text2] = text.split(/(? elem.trim()); + if (text1 === "" && (!text2 || text2 === "")) return false; + return true; + } + + paramsFunc() { const newArgs = this.getOptionString("text") ?? this.args.join(" "); - let [text1, text2] = newArgs - .replaceAll(url ?? "", "") - .split(/(? elem.trim()); - if (!text2?.trim()) text2 = name; + const [text1, text2] = newArgs.split(/(? elem.trim()); const font = this.getOptionString("font"); const phase = this.getOptionString("phase"); return { - caption: text1?.trim() ? this.clean(text1) : random(prompts), + caption: this.clean(text1), caption2: this.clean(text2), path: `assets/images/uncanny/${phase && names.includes(phase.toLowerCase()) ? phase.toLowerCase() : random(names.filter((val) => val !== "goated"))}.png`, // @ts-expect-error this.constructor allows us to get static properties, but TS interprets it as a pure function @@ -79,8 +69,6 @@ class UncannyCommand extends MediaCommand { return this; } - static textOptional = true; - static description = "Makes a Mr. Incredible Becomes Uncanny image (separate left/right text with a comma)"; static aliases = ["canny", "incredible", "pain"]; diff --git a/commands/image-editing/unfreeze.js b/commands/image-editing/unfreeze.js index b00d7a74..8a592d7e 100644 --- a/commands/image-editing/unfreeze.js +++ b/commands/image-editing/unfreeze.js @@ -7,7 +7,6 @@ class UnfreezeCommand extends MediaCommand { static description = "Unfreezes an image sequence"; - static requiresAnim = true; static alwaysGIF = true; static noImage = "You need to provide an image/GIF to unfreeze!"; static command = "freeze"; diff --git a/commands/image-editing/whisper.js b/commands/image-editing/whisper.js index 4b6d5290..adfd5251 100644 --- a/commands/image-editing/whisper.js +++ b/commands/image-editing/whisper.js @@ -1,11 +1,8 @@ import MediaCommand from "#cmd-classes/mediaCommand.js"; class WhisperCommand extends MediaCommand { - /** - * @param {string | undefined} url - */ - paramsFunc(url) { - const newArgs = this.getOptionString("text") ?? this.args.filter((item) => !item.includes(url ?? "")).join(" "); + paramsFunc() { + const newArgs = this.getOptionString("text") ?? this.args.join(" "); return { caption: this.clean(newArgs), }; diff --git a/commands/message/select-image.js b/commands/message/select-image.js index e65b5d9e..47ffd7fa 100644 --- a/commands/message/select-image.js +++ b/commands/message/select-image.js @@ -1,42 +1,32 @@ import { Message } from "oceanic.js"; import Command from "#cmd-classes/command.js"; import { selectedImages } from "#utils/collections.js"; +import { request } from "#utils/media.js"; import imageDetect from "#utils/mediadetect.js"; class SelectImageCommand extends Command { async run() { const message = this.interaction?.data.target; if (!(message instanceof Message)) throw Error("Target is not a message"); - const image = await imageDetect( - this.client, - this.permissions, - ["image"], - message, - this.interaction, - true, - true, - ).catch((e) => { + const mediaArr = await imageDetect(this.client, this.permissions, message, this.interaction, true).catch((e) => { if (e.name === "AbortError") return this.getString("image.timeout"); throw e; }); - if (typeof image === "string") return image; this.success = false; - if (image === undefined) { - return this.getString("image.couldNotFind"); - } - if (image.type === "large") { - return this.getString("image.large"); - } - if (image.type === "tenorlimit") { - return this.getString("image.tenor"); - } - if (image.type === "klipylimit") { - return this.getString("image.klipy"); - } - if (image.type === "badurl") { - return this.getString("image.badurl"); + if (typeof mediaArr === "string") return image; + if (mediaArr.length === 0) return this.getString("image.couldNotFind"); + + let final; + for (const media of mediaArr) { + const type = await request(new URL(media.path), ["image", "audio"], true).catch(() => {}); + if (type) { + final = media; + break; + } } - selectedImages.set(this.author.id, image); + if (!final) return this.getString("image.couldNotFind"); + + selectedImages.set(this.author.id, final); return this.getString("image.selected"); } diff --git a/locales/en-US.json b/locales/en-US.json index e158568f..9a4e4e2a 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -995,7 +995,6 @@ "errorCaps": "ERROR", "guildOnly": "This command only works in servers!", "image": { - "badurl": "That URL is invalid!", "couldNotFind": "I couldn't find an image in that message!", "empty": "The resulting output was empty!", "flagNames": { @@ -1018,7 +1017,6 @@ }, "frames": "That GIF has way too many frames (over 1000)!", "jobEnded": "Something happened to the media servers before I could receive the output. Try running your command again.", - "klipy": "I've been rate-limited by KLIPY. Please try uploading your GIF elsewhere.", "large": "That file is too large (>= 40MB)! Try using a smaller file.", "nocmd": "That command isn't supported on this instance of esmBot.", "noanim": "That image isn't in an animated format (GIF or WEBP)!", @@ -1034,7 +1032,6 @@ "slowDown": "Please slow down a bit.", "tempImageSent": "Here's your image!", "tempSite": "The result was larger than Discord's upload limit, so it was uploaded to an external site instead.", - "tenor": "I've been rate-limited by Tenor. Please try uploading your GIF elsewhere.", "timeout": "The request to get that image timed out. Please try again, upload your image elsewhere, or use another image.", "timeoutDownload": "The request timed out before I could download that image. Try uploading your image somewhere else or reducing its size.", "tip": "(Tip: try right-clicking/holding on a message and press Apps -> {{name}} -> Select Image, then try again.)", diff --git a/natives/common/maps.h b/natives/common/maps.h index 2701938d..2077bcec 100644 --- a/natives/common/maps.h +++ b/natives/common/maps.h @@ -1,3 +1,5 @@ +#include + #include "../image/commands.h" #include "argmap.h" @@ -62,6 +64,12 @@ namespace esmb { {"sonic", &Sonic } }; + const std::vector AnimFunctions = { + "freeze", + "reverse", + "speed", + }; + const std::map FunctionArgsMap = { {"blur", &BlurArgs }, {"caption", &CaptionArgs }, diff --git a/natives/node/media.cc b/natives/node/media.cc index 129a0923..e56e9a2f 100644 --- a/natives/node/media.cc +++ b/natives/node/media.cc @@ -171,12 +171,20 @@ Napi::Object Init(Napi::Env env, Napi::Object exports) { Napi::Array imageFuncs = Napi::Array::New(env); for (auto const &imap : esmb::Image::FunctionMap) { Napi::HandleScope scope(env); - imageFuncs[i] = Napi::String::New(env, imap.first); + Napi::Object func = Napi::Object::New(env); + func.Set("name", Napi::String::New(env, imap.first)); + func.Set("input", Napi::Boolean::From(env, true)); + func.Set("anim", Napi::Boolean::From(env, std::count(esmb::Image::AnimFunctions.begin(), + esmb::Image::AnimFunctions.end(), imap.first) > 0)); + imageFuncs[i] = func; i++; } for (auto const &imap : esmb::Image::NoInputFunctionMap) { Napi::HandleScope scope(env); - imageFuncs[i] = Napi::String::New(env, imap.first); + Napi::Object func = Napi::Object::New(env); + func.Set("name", Napi::String::New(env, imap.first)); + func.Set("input", Napi::Boolean::From(env, false)); + imageFuncs[i] = func; i++; } funcs.Set("image", imageFuncs); diff --git a/src/api/IMPLEMENTATION.md b/src/api/IMPLEMENTATION.md index efc6c796..a0f9ab8c 100644 --- a/src/api/IMPLEMENTATION.md +++ b/src/api/IMPLEMENTATION.md @@ -53,20 +53,19 @@ The job object is formatted like this: ```js { - "type": "image", // type of job to execute - "cmd": string, // name of internal media command, e.g. caption - "path": string, // canonical media URL, used for getting the actual input - "url": string, // original media URL, used for message filtering - "params": { // content varies depending on the command, some common parameters are listed here - "togif": boolean, // convert output to gif + "cmd": string, // name of internal media command, e.g. caption + "inputs": [{ // array of objects describing the input media + "path": string, // canonical media URL, used for getting the actual input + "spoiler": boolean, // whether the input media has a spoiler tag + }], + "params": { // content varies depending on the command, some common parameters are listed here + "togif": boolean, // convert output to gif ... }, - "input": { - "type": string // mime type of output, should usually be the same as input - }, - "name": string, // filename of the media file, without extension - "ephemeral": string, // whether to post the output as an ephemeral message (only when responding directly, see below section) - "spoiler": string // whether to post the output as a spoiler (only when responding directly, see below section) + "id": string, // identifier for the job object + "ephemeral": string, // whether to post the output as an ephemeral message (only when responding directly, see below section) + "spoiler": string, // whether to post the output as a spoiler (only when responding directly, see below section) + "token": string, // the interaction token (only when responding directly, see below section) } ``` diff --git a/src/api/index.ts b/src/api/index.ts index 8019b2ad..bfe46218 100644 --- a/src/api/index.ts +++ b/src/api/index.ts @@ -9,7 +9,7 @@ import { WebSocketServer, type ErrorEvent } from "ws"; import logger from "#utils/logger.js"; import { media } from "#utils/mediaLib.js"; import run from "#utils/mediaRunner.js"; -import type { MediaFormats, MediaParams } from "#utils/types.js"; +import type { JobOutput, MediaFormats, MediaParams } from "#utils/types.js"; const formats = media.init(); @@ -46,6 +46,7 @@ interface Job { error?: string; data?: Buffer; ext?: string; + spoiler?: boolean; } interface MiniJob { @@ -145,7 +146,7 @@ wss.on("connection", (ws, request) => { if (media.funcs.image && formats.image) { cmdFormats.image = {}; for (const cmd of media.funcs.image) { - cmdFormats.image[cmd] = formats.image; + cmdFormats.image[cmd.name] = formats.image; } } const init = Buffer.concat([ @@ -168,13 +169,21 @@ wss.on("connection", (ws, request) => { if (opcode === Tqueue) { const id = msg.readBigInt64LE(3); const obj = msg.subarray(11).toString(); - const job = { msg: JSON.parse(obj), num: jobs.size, verifyEvent: new EventEmitter() }; + const parsed = JSON.parse(obj); + const job = { msg: parsed, num: jobs.size, verifyEvent: new EventEmitter() }; jobs.set(id, job); const newBuffer = Buffer.concat([Buffer.from([Rqueue]), tag]); ws.send(newBuffer); - log(`Got WS request for job ${obj} with id ${id}`, job.num); + log( + `Got WS request for job ${JSON.stringify(parsed, (k, v) => { + if (k === "inputs" && v && v.length > 1) return [v[0], "..."]; + if (k === "token") return null; + return v; + })} with id ${id}`, + job.num, + ); acceptJob(id, ws); } else if (opcode === Tcancel) { jobs.delete(BigInt(req)); @@ -262,6 +271,7 @@ httpServer.on("request", (req, res) => { } if (contentType) res.setHeader("Content-Type", contentType); else res.setHeader("Content-Type", job.ext ?? "application/octet-stream"); + if (job.spoiler) res.setHeader("X-Spoiler", "true"); jobs.delete(id); return res.end(job.data); } @@ -353,16 +363,12 @@ process.on("SIGTERM", (s) => sigHandler(s)); const allowedExtensions = ["gif", "png", "jpeg", "jpg", "webp", "avif"]; const fileSize = 10485760; -async function finishJob( - data: { buffer: Buffer; fileExtension: string }, - job: MiniJob, - object: MediaParams, - ws: WSocket, -) { +async function finishJob(data: JobOutput, job: MiniJob, object: MediaParams, ws: WSocket) { log(`Sending result of job ${job.id}`, job.num); const jobObject = jobs.get(job.id)!; jobObject.data = data.buffer; - jobObject.ext = data.fileExtension; + jobObject.ext = data.type; + jobObject.spoiler = data.spoiler; let tag: Buffer; if (!jobObject.tag) { tag = await waitForVerify(jobObject.verifyEvent); @@ -382,7 +388,7 @@ async function finishJob( form.set( "files[0]", new Blob([jobObject.data]), - `${object.spoiler ? "SPOILER_" : ""}${object.cmd}.${jobObject.ext}`, + `${jobObject.spoiler || object.spoiler ? "SPOILER_" : ""}${object.cmd}.${jobObject.ext}`, ); const controller = new AbortController(); const timeout = setTimeout(() => { @@ -429,21 +435,7 @@ async function finishJob( * Run a media job. */ async function runJob(job: MiniJob, ws: WSocket): Promise { - log(`Job ${job.id} starting...`, job.num); - - const object = job.msg; - - // Any other job type is invalid - if (object.type !== "image") { - throw new TypeError("Unknown job type"); - } - - // If the input has a path, it must also have a type - if (object.path && !object.input?.type) { - throw new TypeError("Unknown media type"); - } - log(`Job ${job.id} started`, job.num); - const data = await run(object); - await finishJob(data, job, object, ws); + const data = await run(job.msg); + await finishJob(data, job, job.msg, ws); } diff --git a/src/classes/mediaCommand.ts b/src/classes/mediaCommand.ts index a618ab9b..9b95c486 100644 --- a/src/classes/mediaCommand.ts +++ b/src/classes/mediaCommand.ts @@ -13,19 +13,19 @@ import { runningCommands, selectedImages } from "#utils/collections.js"; import { convFlagType } from "#utils/handler.js"; import { getAllLocalizations } from "#utils/i18n.js"; import { runMediaJob } from "#utils/media.js"; -import mediaDetect, { type MediaMeta } from "#utils/mediadetect.js"; +import mediaDetect from "#utils/mediadetect.js"; import { clean, isEmpty, random } from "#utils/misc.js"; -import type { ExtendedConstructedCommandOptions, MediaParams } from "#utils/types.js"; +import type { ExtendedConstructedCommandOptions, MediaParams, MediaMeta, MediaTypes } from "#utils/types.js"; import Command from "./command.ts"; class MediaCommand extends Command { params?: object; - paramsFunc(_url?: string, _name?: string): object { + paramsFunc(): object { return {}; } - async criteria(_text?: string | number | boolean | User | Attachment, _url?: string) { + async criteria(_text?: string | number | boolean | User | Attachment) { return true; } @@ -51,68 +51,42 @@ class MediaCommand extends Command { const staticProps = this.constructor as typeof MediaCommand; + const ephemeral = this.getOptionBoolean("ephemeral"); + const spoiler = this.getOptionBoolean("spoiler"); let mediaParams: MediaParams; - let needsSpoiler = false; if (staticProps.requiresImage) { try { let selection: MediaMeta | undefined; if (!this.getOptionAttachment("image") && !this.getOptionString("link")) { selection = selectedImages.get(this.author.id); } - const image = - selection ?? - (await mediaDetect( - this.client, - this.permissions, - staticProps.supportedTypes, - this.message, - this.interaction, - true, - ).catch((e) => { - if (e.name === "AbortError") { - runningCommands.delete(this.author.id); - return this.getString("image.timeout"); - } - throw e; - })); - if (image === undefined) { + const media = selection + ? [selection] + : await mediaDetect(this.client, this.permissions, this.message, this.interaction).catch((e) => { + if (e.name === "AbortError") { + runningCommands.delete(this.author.id); + return this.getString("image.timeout"); + } + throw e; + }); + if (media.length === 0) { runningCommands.delete(this.author.id); return `${this.getString(`commands.noImage.${this.cmdName}`, { returnNull: true }) || this.getString("image.noImage", { returnNull: true }) || staticProps.noImage} ${this.getString("image.tip", { params: { name: this.client.user.globalName ?? this.client.user.username } })}`; } - if (typeof image === "string") return image; + if (typeof media === "string") return media; selectedImages.delete(this.author.id); - needsSpoiler = image.spoiler; - if (image.type === "large") { - runningCommands.delete(this.author.id); - return this.getString("image.large"); - } - if (image.type === "tenorlimit") { - runningCommands.delete(this.author.id); - return this.getString("image.tenor"); - } - if (image.type === "klipylimit") { - runningCommands.delete(this.author.id); - return this.getString("image.klipy"); - } - if (image.type === "badurl") { - runningCommands.delete(this.author.id); - return this.getString("image.badurl"); - } mediaParams = { cmd: staticProps.command, - type: image.mediaType ?? "image", params: { togif: !!this.getOptionBoolean("togif"), - }, - input: { - type: image.type, + ...(this.params ?? this.paramsFunc()), }, id: (this.interaction ?? this.message)?.id ?? Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(), - path: image.path, - url: image.url, // technically not required but can be useful for text filtering - name: image.name, - onlyAnim: !!staticProps.requiresAnim, + inputs: media, + ephemeral, + spoiler, + token: this.interaction?.token, }; } catch (e) { runningCommands.delete(this.author.id); @@ -121,17 +95,18 @@ class MediaCommand extends Command { } else { mediaParams = { cmd: staticProps.command, - type: "image", params: { togif: !!this.getOptionBoolean("togif"), + ...(this.params ?? this.paramsFunc()), }, + inputs: [], id: (this.interaction ?? this.message)?.id ?? Math.floor(Math.random() * Number.MAX_SAFE_INTEGER).toString(), + ephemeral, + spoiler, + token: this.interaction?.token, }; } - const spoiler = this.getOptionBoolean("spoiler"); - if (spoiler != null) needsSpoiler = spoiler; - if (staticProps.requiresParam) { const text = this.getOption( @@ -140,7 +115,7 @@ class MediaCommand extends Command { staticProps.requiredParamType !== Constants.ApplicationCommandOptionTypes.STRING && staticProps.requiredParamType !== "string", ) ?? this.args.join(" ").trim(); - if (!text || (typeof text === "string" && isEmpty(text)) || !(await this.criteria(text, mediaParams.url))) { + if (!text || (typeof text === "string" && isEmpty(text)) || !(await this.criteria(text))) { runningCommands.delete(this.author?.id); return ( this.getString(`commands.noParam.${this.cmdName}`, { returnNull: true }) || @@ -150,31 +125,13 @@ class MediaCommand extends Command { } } - if (this.params) { - Object.assign(mediaParams.params, this.params); - } else { - Object.assign(mediaParams.params, this.paramsFunc(mediaParams.url, mediaParams.name)); - } - let status: Message | undefined; - if ( - mediaParams.input && - (mediaParams.input.type === "image/gif" || mediaParams.input.type === "image/webp") && - this.message - ) { + if (this.message) { status = await this.processMessage( this.message.channel ?? (await this.client.rest.channels.get(this.message.channelID)), ); } - const ephemeral = this.getOptionBoolean("ephemeral"); - - if (this.interaction) { - mediaParams.ephemeral = ephemeral; - mediaParams.spoiler = needsSpoiler; - mediaParams.token = this.interaction.token; - } - try { const result = await runMediaJob(mediaParams); const buffer = result.buffer; @@ -185,21 +142,21 @@ class MediaCommand extends Command { const path = new URL(attachment.proxy_url ?? attachment.proxyURL); path.searchParams.set("animated", "true"); selectedImages.set(this.interaction.user.id, { - url: attachment.url, path: path.toString(), - name: attachment.filename, - type: attachment.content_type ?? attachment.contentType, spoiler: attachment.filename.startsWith("SPOILER_"), }); } return; } + if (type === "large") return this.getString("image.large"); if (type === "frames") return this.getString("image.frames"); if (type === "unknown") return this.getString("image.unknown"); if (type === "noresult") return this.getString("image.noResult"); if (type === "ratelimit") return this.getString("image.ratelimit"); if (type === "nocmd") return this.getString("image.nocmd"); - if (type === "noanim" && staticProps.requiresAnim) return this.getString("image.noanim"); + if (type === "noanim") return this.getString("image.noanim"); + if (type === "nomedia") + return `${this.getString(`commands.noImage.${this.cmdName}`, { returnNull: true }) || this.getString("image.noImage", { returnNull: true }) || staticProps.noImage} ${this.getString("image.tip", { params: { name: this.client.user.globalName ?? this.client.user.username } })}`; if (type === "empty") return staticProps.empty; this.success = true; if (type === "text") @@ -211,7 +168,7 @@ class MediaCommand extends Command { files: [ { contents: buffer, - name: `${needsSpoiler ? "SPOILER_" : ""}${staticProps.command}.${type}`, + name: `${spoiler || result.spoiler ? "SPOILER_" : ""}${staticProps.command}.${type}`, }, ], flags: ephemeral ? 64 : undefined, @@ -315,14 +272,13 @@ class MediaCommand extends Command { "ubuntu", ]; - static supportedTypes: MediaParams["type"][] = ["image"]; + static supportedTypes: MediaTypes[] = ["image"]; static requiresImage = true; static requiresParam = false; static requiredParam = "text"; static requiredParamType: ExtendedConstructedCommandOptions["type"] = "string"; static textOptional = false; - static requiresAnim = false; static alwaysGIF = false; static noImage = "You need to provide an image/GIF!"; static noParam = "You need to provide some text!"; diff --git a/src/events/interactionCreate.ts b/src/events/interactionCreate.ts index 7ebbaf9d..ba18a908 100644 --- a/src/events/interactionCreate.ts +++ b/src/events/interactionCreate.ts @@ -98,10 +98,7 @@ export default async ({ client, database }: EventParams, interaction: AnyInterac const path = new URL(attachment.proxyURL); path.searchParams.set("animated", "true"); selectedImages.set(interaction.user.id, { - url: attachment.url, path: path.toString(), - name: attachment.filename, - type: attachment.contentType, spoiler: attachment.filename.startsWith("SPOILER_"), }); } diff --git a/src/utils/collections.ts b/src/utils/collections.ts index 5085487a..d0cecc35 100644 --- a/src/utils/collections.ts +++ b/src/utils/collections.ts @@ -1,6 +1,5 @@ import type InteractionCollector from "../pagination/awaitinteractions.ts"; -import type { MediaMeta } from "./mediadetect.ts"; -import type { ExtCommand } from "./types.ts"; +import type { ExtCommand, MediaMeta } from "./types.ts"; export const commands = new Map(); export const messageCommands = new Map(); diff --git a/src/utils/media.ts b/src/utils/media.ts index ddf18148..f22843d3 100644 --- a/src/utils/media.ts +++ b/src/utils/media.ts @@ -2,14 +2,14 @@ import { Buffer } from "node:buffer"; import { lookup } from "node:dns/promises"; import fs from "node:fs"; import process from "node:process"; -import { fileTypeFromStream } from "file-type"; +import { fileTypeStream, type AnyWebReadableByteStreamWithFileType } from "file-type"; import ipaddr from "ipaddr.js"; import logger from "./logger.ts"; import MediaConnection from "./mediaConnection.ts"; +import run from "./mediaRunner.ts"; import { random } from "./misc.ts"; -import type { MediaParams, MediaTypeData } from "./types.ts"; +import type { MediaParams, MediaTypes } from "./types.ts"; -const run = process.env.API_TYPE === "ws" ? null : (await import("./mediaRunner.ts")).default; let mediaLib: import("./mediaLib.ts").MediaLib | undefined; interface ServerConfig { @@ -31,11 +31,48 @@ export async function initMediaLib() { mediaLib = media; } -export async function getType( +export async function request( media: URL, - extraReturnTypes: boolean, - typeMedia: MediaParams["type"][], -): Promise { + typeMedia: MediaTypes[], + typeOnly: true, +): Promise< + | { + url: string; + type: string; + mediaType: MediaTypes; + ext: string; + } + | undefined +>; +export async function request( + media: URL, + typeMedia: MediaTypes[], + typeOnly: false, +): Promise< + | { + buf: Buffer; + url: string; + type: string; + mediaType: MediaTypes; + ext: string; + } + | undefined +>; +export async function request( + media: URL, + typeMedia: MediaTypes[], + typeOnly = false, +): Promise< + | { + buf?: Buffer; + url: string; + type: string; + mediaType: MediaTypes; + ext: string; + } + | undefined +> { + // verify that IP address is valid try { const remoteIP = await lookup(media.host); const parsedIP = ipaddr.parse(remoteIP.address); @@ -45,82 +82,88 @@ export async function getType( if ("code" in err && err.code === "ENOTFOUND") return; throw e; } - let type: string | undefined; - let mediaType: MediaParams["type"] | undefined; + let url: string; + let stream: AnyWebReadableByteStreamWithFileType; + + let size = 0; + const controller = new AbortController(); const timeout = setTimeout(() => { controller.abort(); - }, 3000); + }, 15000); try { - const mediaRequest = await fetch(media, { + const res = await fetch(media, { signal: controller.signal, - method: "HEAD", headers: { - "User-Agent": `Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com) esmBot/${process.env.ESMBOT_VER} (+https://esmbot.net)`, + "User-Agent": `Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com) esmBot/${process.env.ESMBOT_VER}`, }, }); clearTimeout(timeout); - if (mediaRequest.redirected) { - const redirectHost = new URL(mediaRequest.url).host; + url = res.url; + if (res.status === 429) throw "ratelimit"; + + if (res.redirected) { + const redirectHost = new URL(res.url).host; const remoteIP = await lookup(redirectHost); const parsedIP = ipaddr.parse(remoteIP.address); if (parsedIP.range() !== "unicast") return; } - url = mediaRequest.url; - let size = 0; - if (mediaRequest.headers.has("content-range")) { - const contentRange = mediaRequest.headers.get("content-range"); + + if (res.headers.has("content-range")) { + const contentRange = res.headers.get("content-range"); if (contentRange) size = Number.parseInt(contentRange.split("/")[1]); - } else if (mediaRequest.headers.has("content-length")) { - const contentLength = mediaRequest.headers.get("content-length"); + } else if (res.headers.has("content-length")) { + const contentLength = res.headers.get("content-length"); if (contentLength) size = Number.parseInt(contentLength); } - if (size > 41943040 && extraReturnTypes) { + + if (size > 41943040) { // 40 MB - type = "large"; - return { type }; - } - const typeHeader = mediaRequest.headers.get("content-type"); - if (typeHeader) { - type = typeHeader; - const typePrefix = typeHeader.split("/")[0]; - if (typePrefix !== "image") return; - mediaType = typePrefix; - } else { - const timeout = setTimeout(() => { - controller.abort(); - }, 3000); - const bufRequest = await fetch(url, { - signal: controller.signal, - headers: { - range: "bytes=0-1023", - "User-Agent": `Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com) esmBot/${process.env.ESMBOT_VER} (+https://esmbot.net)`, - }, - }); - clearTimeout(timeout); - if (bufRequest.body) { - const fileType = await fileTypeFromStream(bufRequest.body); - if (fileType) { - if ( - ![...(typeMedia.length === 0 ? formats.image : typeMedia.flatMap((v) => formats[v]))].includes( - fileType.mime, - ) - ) - return; - - const typePrefix = fileType.mime.split("/")[0] as MediaParams["type"]; - if (!typeMedia.includes(typePrefix)) return; - mediaType = typePrefix; - - if (mediaType) type = fileType.mime; - } - } + throw "large"; } + + if (!res.body) return; + + stream = await fileTypeStream(res.body, { sampleSize: 1024 }); + if (!stream.fileType?.mime) return; } finally { clearTimeout(timeout); } - return { type, url, mediaType }; + + if ( + ![...(typeMedia.length === 0 ? formats.image : typeMedia.flatMap((v) => formats[v]))].includes(stream.fileType.mime) + ) + return; + + const mediaType = stream.fileType.mime.split("/")[0] as MediaTypes; + if (!typeMedia.includes(mediaType)) return; + + const type = stream.fileType.mime; + const ext = stream.fileType.ext; + if (typeOnly) return { url, type, ext, mediaType }; + + const reader = stream.getReader(); + const bufs: Uint8Array[] = []; + let bufSize = 0; + + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + bufs.push(value); + bufSize += value.byteLength; + + if (size && bufSize >= size) break; + + if (bufSize > 41943040) { + // 40 MB + throw "large"; + } + } + + const buf = Buffer.concat(bufs); + return { buf: buf, ext, url, type, mediaType }; } function connect(server: string, auth: string | undefined, name: string | undefined, tls?: boolean) { @@ -176,11 +219,10 @@ async function getIdeal(object: MediaParams): Promise { +export async function runMediaJob(params: MediaParams): Promise<{ buffer: Buffer; type: string; spoiler: boolean }> { if (process.env.API_TYPE === "ws") { const currentServer = await getIdeal(params); if (!currentServer) return { buffer: Buffer.alloc(0), type: "nocmd", + spoiler: false, }; try { await currentServer.queue(BigInt(params.id), params); @@ -211,6 +254,7 @@ export async function runMediaJob(params: MediaParams): Promise<{ buffer: Buffer return { buffer: result.data, type: "sent", + spoiler: false, }; const output = await currentServer.getOutput(params.id); return output; @@ -223,6 +267,7 @@ export async function runMediaJob(params: MediaParams): Promise<{ buffer: Buffer return { buffer: Buffer.alloc(0), type: "noresult", + spoiler: false, }; } if (run) { @@ -235,10 +280,7 @@ export async function runMediaJob(params: MediaParams): Promise<{ buffer: Buffer mediaLib.trim(); } }); - return { - buffer: Buffer.from([...data.buffer]), - type: data.fileExtension, - }; + return data; } throw "media_not_working"; } diff --git a/src/utils/mediaConnection.ts b/src/utils/mediaConnection.ts index 00fdda52..04c9061c 100644 --- a/src/utils/mediaConnection.ts +++ b/src/utils/mediaConnection.ts @@ -3,7 +3,7 @@ import { setTimeout as setTimeoutPromise } from "node:timers/promises"; import WSocket, { type Data, type ErrorEvent } from "ws"; import logger from "./logger.ts"; import { mimeToExt } from "./mime.ts"; -import type { MediaFormats, MediaFuncs } from "./types.ts"; +import type { JobOutput, MediaFormats, MediaFuncs, MediaFuncTypes, MediaTypes } from "./types.ts"; const Rerror = 0x01; const Tqueue = 0x02; @@ -25,7 +25,7 @@ interface RequestState { type WaitResponse = { sent: true; data: Buffer } | { sent: false }; -const etcTypes = ["noanim", "nocmd", "text", "empty", "frames", "ratelimit"]; +const etcTypes = ["noanim", "nocmd", "text", "empty", "frames", "ratelimit", "nomedia"]; class MediaConnection { requests: Map; @@ -36,6 +36,7 @@ class MediaConnection { disconnected: boolean; formats: MediaFormats; funcs: MediaFuncs; + types: MediaFuncTypes; wsproto: string; sockurl: string; conn: WSocket; @@ -50,6 +51,7 @@ class MediaConnection { this.disconnected = false; this.formats = {}; this.funcs = {}; + this.types = {}; if (tls) { this.wsproto = "wss"; } else { @@ -83,6 +85,14 @@ class MediaConnection { this.funcs = { image: this.formats.image ? Object.keys(this.formats.image) : [], }; + this.types = {}; + for (const [type, cmdList] of Object.entries(this.formats)) { + const cmds = Object.keys(cmdList); + for (const cmd of cmds) { + if (!this.types[cmd]) this.types[cmd] = []; + this.types[cmd].push(type as MediaTypes); + } + } return; } if (op === Rclose) { @@ -166,7 +176,7 @@ class MediaConnection { return this.do(Tcancel, jobid, buf); } - async getOutput(jobid: string) { + async getOutput(jobid: string): Promise { logger.debug(`Getting output of ${jobid} on media server ${this.host}`); const req = await fetch( `${this.httpurl}/media?id=${jobid}`, @@ -181,7 +191,7 @@ class MediaConnection { const contentType = req.headers.get("content-type"); let type = contentType ? mimeToExt(contentType) : "unknown"; if (type === "unknown" && contentType && etcTypes.includes(contentType)) type = contentType; - return { buffer: Buffer.from(await req.arrayBuffer()), type }; + return { buffer: Buffer.from(await req.arrayBuffer()), type, spoiler: req.headers.has("X-Spoiler") }; } async getCount() { diff --git a/src/utils/mediaLib.ts b/src/utils/mediaLib.ts index 47357baa..6d89a94e 100644 --- a/src/utils/mediaLib.ts +++ b/src/utils/mediaLib.ts @@ -1,18 +1,27 @@ import { Buffer } from "node:buffer"; import { createRequire } from "node:module"; import process from "node:process"; -import type { MediaParams } from "./types.ts"; +import type { MediaParams, MediaTypes } from "./types.ts"; + +interface FuncObject { + name: string; + input: boolean; + anim?: boolean; +} export interface MediaLib { funcs: { - image: string[]; + image: FuncObject[]; }; process( - type: MediaParams["type"], + type: MediaTypes, cmd: string, params: MediaParams["params"], - input: MediaParams["input"], + input: { + data?: ArrayBuffer; + type?: string; + }, ): Promise<{ data: Buffer; type: string }>; init(): { image?: string[]; diff --git a/src/utils/mediaRunner.ts b/src/utils/mediaRunner.ts index c7100a47..52d2a008 100644 --- a/src/utils/mediaRunner.ts +++ b/src/utils/mediaRunner.ts @@ -1,74 +1,128 @@ import { Buffer } from "node:buffer"; import path from "node:path"; import { fileURLToPath } from "node:url"; -import { media } from "./mediaLib.ts"; +import { request } from "./media.ts"; +import type { MediaLib } from "./mediaLib.ts"; import { mimeToExt } from "./mime.ts"; -import type { MediaParams } from "./types.ts"; +import type { JobOutput, MediaParams, MediaTypes } from "./types.ts"; + +let media: MediaLib | undefined; const defaultExts = { image: "png", }; -export default async function run(object: MediaParams): Promise<{ buffer: Buffer; fileExtension: string }> { +export default async function run(object: MediaParams): Promise { + // dynamically load media library + if (!media) { + const imported = await import("./mediaLib.js"); + media = imported.media; + } + // Check if command exists - if (!media.funcs[object.type]?.includes(object.cmd)) { + const supportedTypes: MediaTypes[] = []; + for (const [type, cmds] of Object.entries(media.funcs)) { + if (cmds.some((v) => v.name === object.cmd)) { + supportedTypes.push(type as MediaTypes); + } + } + + if (supportedTypes.length === 0) { return { buffer: Buffer.alloc(0), - fileExtension: "nocmd", + type: "nocmd", + spoiler: false, }; } - let inputBuffer: ArrayBuffer | null = null; - if (object.path) { - // If the file has a path, it must also have a type - if ( - object.type === "image" && - object.input?.type !== "image/gif" && - object.input?.type !== "image/webp" && - object.onlyAnim - ) { - return { - buffer: Buffer.alloc(0), - fileExtension: "noanim", - }; + let inputBuffer: Buffer | undefined; + let fileType: string | undefined; + let mediaType: MediaTypes | undefined; + let spoiler = false; + try { + for (const media of object.inputs) { + const res = await request(new URL(media.path), supportedTypes, false); + if (res) { + inputBuffer = res.buf; + fileType = res.type; + mediaType = res.mediaType; + spoiler = media.spoiler; + break; + } } + if (!inputBuffer || !mediaType) throw "nomedia"; + } catch (e) { + if (typeof e !== "string") throw e; + return { + buffer: Buffer.alloc(0), + type: e, + spoiler: false, + }; + } + + if (object.spoiler) spoiler = true; - const controller = new AbortController(); - const timeout = setTimeout(() => { - controller.abort(); - }, 15000); - try { - const res = await fetch(object.path, { - signal: controller.signal, - headers: { - "User-Agent": `Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com) esmBot/${process.env.ESMBOT_VER} (+https://esmbot.net)`, - }, - }); - clearTimeout(timeout); - if (res.status === 429) throw "ratelimit"; - inputBuffer = await res.arrayBuffer(); - } catch (e) { - if (typeof e !== "string") throw e; - return { - buffer: Buffer.alloc(0), - fileExtension: e, - }; + let validInput = false; + for (const type of supportedTypes) { + const func = media.funcs[type].find((v) => v.name === object.cmd); + if (!func) continue; + + if (func.input) { + if (type === mediaType) { + validInput = true; + break; + } + } else { + validInput = true; + break; } } + // Reject non-animated formats for commands that only work on animations + if ( + supportedTypes.includes("image") && + fileType !== "image/gif" && + fileType !== "image/webp" && + media.funcs.image.find((v) => v.name === object.cmd)?.anim + ) { + return { + buffer: Buffer.alloc(0), + type: "noanim", + spoiler: false, + }; + } + + if (!validInput) { + return { + buffer: Buffer.alloc(0), + type: "nomedia", + spoiler: false, + }; + } + + if (!mediaType) { + // A function without input will always return a single media type + mediaType = supportedTypes[0]; + } + // Convert from a MIME type (e.g. "image/png") to something the media processor understands (e.g. "png"). // Don't set `type` directly on the object we are passed as it will be read afterwards. - // If no type is given (say, the command generates its own output), make it a PNG. - const fileExtension = object.input?.type ? mimeToExt(object.input.type) : defaultExts[object.type]; + // If no type is given (say, the command generates its own output), use a default type. + const fileExtension = fileType ? mimeToExt(fileType) : defaultExts[mediaType]; + + const inputObj = { + data: inputBuffer.buffer.slice( + inputBuffer.byteOffset, + inputBuffer.byteOffset + inputBuffer.byteLength, + ) as ArrayBuffer, + type: fileExtension, + }; - if (object.input) { - if (inputBuffer) object.input.data = inputBuffer; - object.input.type = fileExtension; - } object.params.basePath = path.join(path.dirname(fileURLToPath(import.meta.url)), "../../"); - const { data, type } = await media.process(object.type, object.cmd, object.params, object.input ?? {}); + const { data, type } = await media.process(mediaType, object.cmd, object.params, inputBuffer ? inputObj : {}); return { buffer: data, - fileExtension: type, + type, + spoiler, }; } diff --git a/src/utils/mediadetect.ts b/src/utils/mediadetect.ts index ad68c169..dd4f64a4 100644 --- a/src/utils/mediadetect.ts +++ b/src/utils/mediadetect.ts @@ -13,8 +13,7 @@ import { ThreadChannel, } from "oceanic.js"; import logger from "./logger.ts"; -import { formats, getType } from "./media.ts"; -import type { MediaParams, MediaTypeData } from "./types.ts"; +import type { MediaMeta } from "./types.ts"; const tenorURLs = ["tenor.com", "www.tenor.com"]; const giphyURLs = ["giphy.com", "www.giphy.com", "i.giphy.com"]; @@ -33,6 +32,9 @@ const combined = [...tenorURLs, ...giphyURLs, ...giphyMediaURLs, ...klipyURLs]; const providerUrls = ["https://tenor.co", "https://tenor.com", "https://giphy.com", "https://klipy.com"]; +const discordCDNDomains = ["cdn.discordapp.com", "media.discordapp.net"]; +const discordProxyDomains = ["images-ext-1.discordapp.net", "images-ext-2.discordapp.net"]; + type TenorMediaObject = { url: string; dims: number[]; @@ -91,54 +93,25 @@ type KlipyResponse = { }; }; -export type MediaMeta = { - path: string; - type?: string; - url: string; - name: string; - spoiler: boolean; - mediaType?: MediaParams["type"]; -}; - /** * Gets proper media paths. */ async function getMedia( media: string, media2: string, - mediaType: MediaParams["type"][], - single = false, spoiler = false, - extraReturnTypes = false, - type: string | null = null, - client: Client | undefined = undefined, + client?: Client, ): Promise { - let mediaURL: URL; - try { - mediaURL = new URL(media); - if (!mediaURL.host) throw null; - if (mediaURL.protocol !== "http:" && mediaURL.protocol !== "https:") throw null; - } catch { - return { - url: media2, - path: media, - name: "null", - type: "badurl", - spoiler, - }; - } - const fileNameSplit = mediaURL.pathname.split("/"); - const fileName = fileNameSplit[fileNameSplit.length - 1]; - const fileNameNoExtension = fileName.slice(0, fileName.lastIndexOf(".")); + const mediaURL = new URL(media); + if (!mediaURL.host) return; + if (mediaURL.protocol !== "http:" && mediaURL.protocol !== "https:") return; const payload: MediaMeta = { - url: media2, path: media, - name: fileNameNoExtension, spoiler, }; const url2 = new URL(media2); const host = url2.host; - if (mediaType.includes("image") && combined.includes(host)) { + if (combined.includes(host)) { if (tenorURLs.includes(host)) { // Tenor's API tends to be the most reliable way to get a raw GIF, // so we use that if there's a key in the config @@ -156,12 +129,7 @@ async function getMedia( const data = await fetch( `https://tenor.googleapis.com/v2/posts?media_filter=gif&limit=1&client_key=esmBot%20${process.env.ESMBOT_VER}&key=${process.env.TENOR}&ids=${id}`, ); - if (data.status === 429) { - if (extraReturnTypes) { - payload.type = "tenorlimit"; - return payload; - } - } + if (data.status === 429) return; const json = (await data.json()) as TenorResponse; if (json.error) throw Error(json.error.message); if (json.results.length === 0) return; @@ -170,48 +138,41 @@ async function getMedia( const tenorURL = url2; if (!tenorURL.pathname.endsWith(".gif")) tenorURL.pathname += ".gif"; - const redirectReq = await fetch(tenorURL, { method: "HEAD", redirect: "manual" }); + const redirectReq = await fetch(tenorURL, { + method: "HEAD", + redirect: "manual", + }); if (redirectReq.status !== 301 && redirectReq.status !== 302) return; const redirect = redirectReq.headers.get("location"); if (!redirect) return; + payload.path = redirect; } else { return; } - payload.type = "image/gif"; - payload.mediaType = "image"; } else if (klipyURLs.includes(host)) { if (!process.env.KLIPY || process.env.KLIPY === "") return; if (!media2.includes("klipy.com/gifs/")) return; + const id = url2.pathname.replace("/gifs/", ""); const data = await fetch(`https://api.klipy.com/api/v1/${process.env.KLIPY}/gifs/items?slugs=${id}`); - if (data.status === 429) { - if (extraReturnTypes) { - payload.type = "klipylimit"; - return payload; - } - } + if (data.status === 429) return; + const json = (await data.json()) as KlipyResponse; if (json.errors) throw AggregateError(json.errors.message); if (json.data.data.length === 0) return; + payload.path = json.data.data[0].file.hd.gif.url; - payload.type = "image/gif"; - payload.mediaType = "image"; } else if (giphyURLs.includes(host)) { // Can result in an HTML page instead of a WEBP payload.path = `https://media0.giphy.com/media/${media2.split("/")[4].split("-").pop()}/giphy.webp`; - payload.type = "image/webp"; - payload.mediaType = "image"; } else if (giphyMediaURLs.includes(host)) { payload.path = `https://media0.giphy.com/media/${media2.split("/")[4]}/giphy.webp`; - payload.type = "image/webp"; - payload.mediaType = "image"; } } else { - let result: MediaTypeData | undefined; if ( - (mediaURL.host === "cdn.discordapp.com" || mediaURL.host === "media.discordapp.net") && + discordCDNDomains.includes(mediaURL.host) && mediaURL.pathname.match(/^\/(?:ephemeral-)?attachments\/\d+\/\d+\//) ) { let url: URL; @@ -221,29 +182,11 @@ async function getMedia( } else { url = new URL(media); } - if (mediaType.includes("image")) url.searchParams.set("animated", "true"); - result = await getType(url, extraReturnTypes, mediaType); - } else if ( - mediaType.includes("image") && - (mediaURL.host === "images-ext-1.discordapp.net" || mediaURL.host === "images-ext-2.discordapp.net") && - mediaURL.pathname.match(/^\/external\/[\w-]+\//) - ) { + url.searchParams.set("animated", "true"); + } else if (discordProxyDomains.includes(mediaURL.host) && mediaURL.pathname.match(/^\/external\/[\w-]+\//)) { mediaURL.searchParams.set("animated", "true"); - result = await getType(mediaURL, extraReturnTypes, mediaType); - } else { - result = await getType(mediaURL, extraReturnTypes, mediaType); } - if (!result) return; - if (result.url) payload.path = result.url; - payload.type = type ?? result.type; - if (result.mediaType) payload.mediaType = result.mediaType; - if (payload.type === "large" && single) return payload; - if ( - !payload.type || - !payload.mediaType || - ![...(mediaType.length === 0 ? formats.image : mediaType.flatMap((v) => formats[v]))].includes(payload.type) - ) - return; + payload.path = mediaURL.toString(); } return payload; } @@ -251,116 +194,107 @@ async function getMedia( /** * Checks a single message for media */ -async function checkMedia( - message: Message, - extraReturnTypes: boolean, - mediaType: MediaParams["type"][], - singleMessage = false, -): Promise { - let type: MediaMeta | undefined; +async function checkMedia(message: Message): Promise { + const types: MediaMeta[] = []; // first check the embeds if (message.embeds.length !== 0) { - type = await checkEmbeds(message, extraReturnTypes, mediaType, singleMessage); + const type = await checkEmbeds(message); + if (type) types.push(...type); } // then check the components if (message.components.length !== 0) { - type = await checkComponents(message.components, extraReturnTypes, mediaType, singleMessage); + const type = await checkComponents(message.components); + if (type) types.push(...type); } // then check the attachments - if (!type && message.attachments.size !== 0) { + if (message.attachments.size !== 0) { const firstAttachment = message.attachments.first(); - if (firstAttachment) - type = await getMedia( + if (firstAttachment) { + const type = await getMedia( firstAttachment.proxyURL, firstAttachment.url, - mediaType, - singleMessage, !!(firstAttachment.flags & Constants.AttachmentFlags.IS_SPOILER), ); + if (type) types.push(type); + } } // then check embeds, components, and attachments inside forwards - if (!type && message.messageSnapshots?.[0]) { + if (message.messageSnapshots?.[0]) { const forward = message.messageSnapshots?.[0].message; - if (forward.embeds.length !== 0) type = await checkEmbeds(forward, extraReturnTypes, mediaType, singleMessage); - if (forward.components.length !== 0) - type = await checkComponents(forward.components, extraReturnTypes, mediaType, singleMessage); + if (forward.embeds.length !== 0) { + const type = await checkEmbeds(forward); + if (type) types.push(...type); + } + if (forward.components.length !== 0) { + const type = await checkComponents(forward.components); + if (type) types.push(...type); + } - if (!type && forward.attachments.length !== 0) { - type = await getMedia( + if (forward.attachments.length !== 0) { + const type = await getMedia( forward.attachments[0].proxyURL, forward.attachments[0].url, - mediaType, - singleMessage, !!(forward.attachments[0].flags & Constants.AttachmentFlags.IS_SPOILER), ); + if (type) types.push(type); } } // if the return value exists then return it - return type; + return types; } -function checkComponents( - components: MessageComponent[], - extraReturnTypes: boolean, - mediaType: MediaParams["type"][], - singleMessage = false, -) { +async function checkComponents(components: MessageComponent[]) { + const arr = []; + for (const component of components) { // full-size image/video if (component.type === Constants.ComponentTypes.MEDIA_GALLERY) { - return getMedia( + const media = await getMedia( component.items[0].media.proxyURL ?? component.items[0].media.url, component.items[0].media.url, - mediaType, - singleMessage, component.items[0].spoiler, - extraReturnTypes, ); + if (media) arr.push(media); } // section thumbnail if ( component.type === Constants.ComponentTypes.SECTION && component.accessory.type === Constants.ComponentTypes.THUMBNAIL ) { - return getMedia( + const media = await getMedia( component.accessory.media.proxyURL ?? component.accessory.media.url, component.accessory.media.url, - mediaType, - singleMessage, component.accessory.spoiler, - extraReturnTypes, ); + if (media) arr.push(media); } // raw file if (component.type === Constants.ComponentTypes.FILE) { - return getMedia( + const media = await getMedia( component.file.proxyURL ?? component.file.url, component.file.url, - mediaType, - singleMessage, component.spoiler, - extraReturnTypes, ); + if (media) arr.push(media); } } + return arr; } -function checkEmbeds( - message: Message | MessageSnapshotMessage, - extraReturnTypes: boolean, - mediaType: MediaParams["type"][], - singleMessage = false, -) { +async function checkEmbeds(message: Message | MessageSnapshotMessage) { let hasSpoiler = false; if (message.embeds[0].url && message.content) { const spoilerRegex = /\|\|.*https?:\/\/.*\|\|/s; hasSpoiler = spoilerRegex.test(message.content); } + + const arr = []; + // embeds can vary in types, we check for gifvs first if ( message.embeds[0].provider?.url && @@ -369,35 +303,27 @@ function checkEmbeds( message.embeds[0].url && (message.embeds[0].provider.url === "https://klipy.com" ? process.env.KLIPY && process.env.KLIPY !== "" : true) ) { - return getMedia( - message.embeds[0].video.url, - message.embeds[0].url, - mediaType, - singleMessage, - hasSpoiler, - extraReturnTypes, - ); - // then thumbnails + const media = await getMedia(message.embeds[0].video.url, message.embeds[0].url, hasSpoiler); + if (media) arr.push(media); } else if (message.embeds[0].thumbnail) { - return getMedia( + // then thumbnails + const media = await getMedia( message.embeds[0].thumbnail.proxyURL ?? message.embeds[0].thumbnail.url, message.embeds[0].thumbnail.url, - mediaType, - singleMessage, hasSpoiler, - extraReturnTypes, ); - // and finally direct images + if (media) arr.push(media); } else if (message.embeds[0].image) { - return getMedia( + // and finally direct images + const media = await getMedia( message.embeds[0].image.proxyURL ?? message.embeds[0].image.url, message.embeds[0].image.url, - mediaType, - singleMessage, hasSpoiler, - extraReturnTypes, ); + if (media) arr.push(media); } + + return arr; } /** @@ -460,31 +386,30 @@ export async function stickerDetect( export default async ( client: Client, perms: Permission, - mediaType: MediaParams["type"][], cmdMessage?: Message, interaction?: CommandInteraction, - extraReturnTypes = false, singleMessage = false, -): Promise => { +): Promise => { + const arr: MediaMeta[] = []; // we start by determining whether or not we're dealing with an interaction or a message if (interaction) { // we can get a raw attachment or a URL in the interaction itself - const attachment = interaction.data.options.getAttachment(mediaType.length === 1 ? mediaType[0] : "media"); + const attachment = interaction.data.options.getAttachment("image"); if (attachment) { - return getMedia( + const media = await getMedia( attachment.proxyURL, attachment.url, - mediaType, - true, !!(attachment.flags & Constants.AttachmentFlags.IS_SPOILER), - !!attachment.contentType, ); + if (media) arr.push(media); } const link = interaction.data.options.getString("link"); if (link) { - return getMedia(link, link, mediaType, true, false, extraReturnTypes, null, interaction.client); + const media = await getMedia(link, link, false, interaction.client); + if (media) arr.push(media); } } + if (arr.length > 0) return arr; if (cmdMessage) { // check if the message is a reply to another message if (cmdMessage.messageReference?.channelID && cmdMessage.messageReference.messageID && !singleMessage) { @@ -492,14 +417,15 @@ export default async ( .getMessage(cmdMessage.messageReference.channelID, cmdMessage.messageReference.messageID) .catch(() => undefined); if (replyMessage) { - const replyResult = await checkMedia(replyMessage, extraReturnTypes, mediaType); - if (replyResult) return replyResult; + const replyResult = await checkMedia(replyMessage); + if (replyResult) arr.push(...replyResult); } } // then we check the current message - const result = await checkMedia(cmdMessage, extraReturnTypes, mediaType, singleMessage); - if (result) return result; + const result = await checkMedia(cmdMessage); + if (result) arr.push(...result); } + if (arr.length > 0) return arr; if (!singleMessage && (cmdMessage || interaction?.authorizingIntegrationOwners?.[0] !== undefined)) { // if there aren't any replies or interaction attachments then iterate over the last few messages in the channel const context = interaction ?? cmdMessage; @@ -514,13 +440,17 @@ export default async ( !(channel instanceof ThreadChannel) && !(channel instanceof PrivateChannel) ) - return; - if (interaction?.authorizingIntegrationOwners?.[0] !== "0" && perms && !perms.has("READ_MESSAGE_HISTORY")) return; - const messages = await channel.getMessages(); + return arr; + if (interaction?.authorizingIntegrationOwners?.[0] !== "0" && perms && !perms.has("READ_MESSAGE_HISTORY")) + return arr; + const messages = await channel.getMessages({ limit: 50 }); // iterate over each message for (const message of messages) { - const result = await checkMedia(message, extraReturnTypes, mediaType); - if (result) return result; + const result = await checkMedia(message); + if (result.length > 0) { + arr.push(...result); + } } } + return arr; }; diff --git a/src/utils/tempimages.ts b/src/utils/tempimages.ts index 8ef67bb8..74da1504 100644 --- a/src/utils/tempimages.ts +++ b/src/utils/tempimages.ts @@ -11,7 +11,7 @@ import { import { selectedImages } from "./collections.ts"; import { getString } from "./i18n.ts"; import logger from "./logger.ts"; -import { getType } from "./media.ts"; +import { request } from "./media.ts"; type SizeSuffix = "K" | "M" | "G" | "T"; type FileStats = { @@ -75,12 +75,9 @@ export async function upload( ); } if (save) { - const type = await getType(new URL(imageURL), true, []); + const type = await request(new URL(imageURL), [], true).catch(() => {}); selectedImages.set(authorId, { - url: imageURL, path: type?.url ?? imageURL, - name: filename, - type: type?.type, spoiler: result.name.startsWith("SPOILER_"), }); } diff --git a/src/utils/types.ts b/src/utils/types.ts index 12a7718b..93f7a5d7 100644 --- a/src/utils/types.ts +++ b/src/utils/types.ts @@ -65,30 +65,29 @@ export type Param = } | string; +export type MediaTypes = "image"; + export interface MediaParams { cmd: string; - type: "image"; params: { [key: string]: string | number | boolean; }; - input?: { - data?: ArrayBuffer; - type?: string; - }; id: string; - path?: string; - url?: string; - name?: string; - onlyAnim?: boolean; + inputs: MediaMeta[]; ephemeral?: boolean; spoiler?: boolean; token?: string; } -export interface MediaTypeData { - url?: string; - type?: string; - mediaType?: MediaParams["type"]; +export interface MediaMeta { + path: string; + spoiler: boolean; +} + +export interface JobOutput { + buffer: Buffer; + type: string; + spoiler: boolean; } export interface MediaFormats { @@ -101,6 +100,10 @@ export interface MediaFuncs { image?: string[]; } +export interface MediaFuncTypes { + [cmd: string]: MediaTypes[]; +} + export interface SearXNGResults { query: string; results: { -- 2.51.2