import { imageDimensionsFromStream } from "image-dimensions"; import truncate from "just-truncate"; import { v4 as uuidv4 } from "uuid"; import { ImageResizeResult } from "../enums"; import { BSKY_GIF_MIME_TYPES, BSKY_IMG_MIME_TYPES, BSKY_IMG_SIZE_LIMIT, BSKY_IMG_SIZE_LIMIT_IN_MB, BSKY_VIDEO_MIME_TYPES, CF_IMAGES_MAX_DIMENSION, GIF_UPLOAD_ALLOWED, MAX_IMAGE_WIDTH, MB_TO_BYTES, } from "../limits"; import type { AllContext, EmbedData, R2BucketObject, UserIdType } from "../types"; import { addFileListing, deleteFileListings, isMediaOwnedByUser } from "./db/file"; import { isAltEditableType } from "./helpers"; interface FileMetaData { name: string; size: number; user: string; type: string; qualityLevel?: number; height?: number; width?: number; } export const deleteEmbedsFromR2 = async (c: AllContext, embeds: EmbedData[] | undefined, isQueued: boolean = false) => { const itemsToDelete: string[] = []; if (embeds !== undefined && embeds.length > 0) { embeds.forEach((data) => { // We don't store any data locally for weblinks/records if (isAltEditableType(data.type)) { console.log(`Pushing ${data.content} for deletion...`); itemsToDelete.push(data.content.toLowerCase()); } }); await deleteFromR2(c, itemsToDelete, isQueued); } return itemsToDelete; }; export const requestDeleteFromR2 = async (c: AllContext, embedKey: string) => { const checkOwnership = await isMediaOwnedByUser(c, embedKey); if (checkOwnership) { await deleteFromR2(c, embedKey, false); return true; } return false; }; export const deleteFromR2 = async (c: AllContext, embeds: string[]|string, isQueued: boolean=false) => { if (embeds.length <= 0) return; console.log(`Deleting ${embeds.toString()}`); const killFilesPromise = c.env.R2.delete(embeds); const deleteFileListingPromise = deleteFileListings(c, embeds); const deleteFilePromises = Promise.all([killFilesPromise, deleteFileListingPromise]); if (isQueued) { await deleteFilePromises; } else { c.executionCtx.waitUntil(deleteFilePromises); } }; const rawUploadToR2 = async (c: AllContext, buffer: ArrayBuffer | ReadableStream, metaData: FileMetaData) => { const fileExt: string | undefined = metaData.name.split(".").pop(); if (fileExt === undefined) { return { success: false, error: "unable to upload, file name is invalid" }; } const fileName = `${uuidv4()}.${fileExt.toLowerCase()}`; const R2Metadata: Record = { user: metaData.user, type: metaData.type, ext: fileExt, filename: truncate(metaData.name, 500), }; // add width and height metadata if they exist if (metaData.width !== undefined) { R2Metadata.width = metaData.width.toString(); } if (metaData.height !== undefined) { R2Metadata.height = metaData.height.toString(); } // push file const R2UploadRes = await c.env.R2.put(fileName, buffer, { customMetadata: R2Metadata }); // successfully uploaded file if (R2UploadRes.size > 0) { // push file listing await addFileListing(c, fileName, metaData.user); return { success: true, data: R2UploadRes.key, originalName: metaData.name, fileSize: metaData.size, qualityLevel: metaData.qualityLevel, }; } else { return { success: false, error: "unable to push to file storage" }; } }; const uploadImageToR2 = async (c: AllContext, file: File, userId: string) => { const originalName = file.name; // We need to double check this image for various size information. const imageMetaData = await imageDimensionsFromStream(file.stream() as ReadableStream); if (imageMetaData === undefined) { return { success: false, error: "image data could not be processed" }; } // if the image is over the cf image transforms, then return an error. if (imageMetaData.width > CF_IMAGES_MAX_DIMENSION || imageMetaData.height > CF_IMAGES_MAX_DIMENSION) { return { success: false, error: "image dimensions are too large to autosize. make sure your files fit the limits." }; } // our process data once handled. const fileProcessData = { qualityLevel: 100, type: file.type, size: file.size, width: imageMetaData.width, height: imageMetaData.height, }; // The file we'll eventually upload to R2 (this object will change based on compression/resizes) let fileToProcess: ArrayBuffer | ReadableStream | null = null; if (file.size > BSKY_IMG_SIZE_LIMIT) { // reset, we'll get this later. fileProcessData.height = fileProcessData.width = 0; let resizeResult: ImageResizeResult = ImageResizeResult.TooLarge; // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition if (c.env.IMAGE_SETTINGS.enabled && c.env.IMAGE_SETTINGS.steps !== undefined) { // Randomly generated id to be used during the resize process const resizeFilename = uuidv4(); const resizeBucketPush = await c.env.R2RESIZE.put(resizeFilename, await file.bytes(), { customMetadata: { user: userId }, httpMetadata: { contentType: file.type }, }); if (resizeBucketPush.size <= 0) { console.error(`Failed to push ${file.name} to the resizing bucket`); return { success: false, error: "resize process ran out of disk space, you'll need to resize the image or try again" }; } // Default image rules for resizing an image const imageRules: RequestInitCfPropertiesImage = { fit: "scale-down", metadata: "copyright", format: "jpeg" }; // if the application is to also resize the width automatically, do so here. // this will preserve aspect ratio if (MAX_IMAGE_WIDTH > 0) { imageRules.width = MAX_IMAGE_WIDTH; } // eslint-disable-next-line @typescript-eslint/prefer-for-of for (let i = 0; i < c.env.IMAGE_SETTINGS.steps.length; ++i) { const qualityLevel: number = c.env.IMAGE_SETTINGS.steps[i]; const response = await fetch(new URL(resizeFilename, c.env.IMAGE_SETTINGS.bucket_url), { headers: { "x-skyscheduler-helper": c.env.RESIZE_SECRET_HEADER }, cf: { image: { quality: qualityLevel, ...imageRules } }, }); if (response.ok) { const resizedHeader = response.headers.get("Cf-Resized"); const returnType = response.headers.get("Content-Type") ?? ""; const transformFileSize: number = Number(response.headers.get("Content-Length")) || 0; const hasResizeHeader: boolean = resizedHeader !== null; const resizeHadError: boolean = !hasResizeHeader || resizedHeader!.includes("err="); if (!resizeHadError && BSKY_IMG_MIME_TYPES.includes(returnType)) { console.log(`Attempting quality level ${qualityLevel}% for ${originalName}, size: ${transformFileSize}`); // If we make the file size less than the actual limit if (transformFileSize < BSKY_IMG_SIZE_LIMIT && transformFileSize !== 0) { console.log(`${originalName}: Quality level ${qualityLevel}% processed, fits correctly with size.`); resizeResult = ImageResizeResult.Success; // Set some extra variables fileProcessData.qualityLevel = qualityLevel; fileProcessData.size = transformFileSize; fileProcessData.type = "image/jpeg"; // split the streams so we can read the response body more than once const streams = response.body!.tee(); fileToProcess = streams[0]; // attempt to pull the image dimensions const optimizedData = await imageDimensionsFromStream(streams[1] as ReadableStream); fileProcessData.width = optimizedData?.width ?? 0; fileProcessData.height = optimizedData?.height ?? 0; // dispose. await streams[1].cancel(); break; } else { if (transformFileSize !== 0) { // Print how over the image was if we cannot properly resize it console.log(`${originalName}: file size ${transformFileSize - BSKY_IMG_SIZE_LIMIT} over the appropriate size`); } // dispose the body await response.body?.cancel(); } } else { console.warn(`File ${file.name} was not handled ${response.statusText}`); if (hasResizeHeader) { // figure out if we have exhausted image transformations const errCode = /err=(\d+)/.exec(resizedHeader!); if (errCode && parseInt(errCode[1]) === 9422) { resizeResult = ImageResizeResult.ExhaustedResources; break; } } } } else { console.error(`image tf service got error: ${response.statusText}`); } } // Delete the file from the resize bucket. c.executionCtx.waitUntil(c.env.R2RESIZE.delete(resizeFilename)); } if (resizeResult != ImageResizeResult.Success) { let errorString: string; switch (resizeResult) { case ImageResizeResult.ExhaustedResources: errorString = `Image resize service is exhausted, please manually resize the image to be under ${BSKY_IMG_SIZE_LIMIT_IN_MB}MB.`; break; default: case ImageResizeResult.TooLarge: { const fileSizeOverAmount: string = ((file.size - BSKY_IMG_SIZE_LIMIT) / MB_TO_BYTES).toFixed(2); errorString = `Image is too large to post, the file size is over by ${fileSizeOverAmount}MB`; break; } } return { success: false, originalName: originalName, error: errorString }; } } const fileMetaData: FileMetaData = { name: originalName, size: fileProcessData.size, user: userId, type: fileProcessData.type, qualityLevel: fileProcessData.qualityLevel, width: fileProcessData.width > 0 ? fileProcessData.width : undefined, height: fileProcessData.height > 0 ? fileProcessData.height : undefined, }; fileToProcess ??= await file.arrayBuffer(); return rawUploadToR2(c, fileToProcess, fileMetaData); }; const uploadVideoToR2 = async (c: AllContext, file: File, userId: string) => { const fileMetaData: FileMetaData = { name: file.name, size: file.size, type: file.type, user: userId }; return rawUploadToR2(c, file.stream(), fileMetaData); }; export const uploadFileR2 = async (c: AllContext, file: File, userId: UserIdType) => { const fileType: string = file.type.toLowerCase(); if (userId !== null) { if (BSKY_IMG_MIME_TYPES.includes(fileType)) { return uploadImageToR2(c, file, userId); } else if (BSKY_VIDEO_MIME_TYPES.includes(fileType)) { return uploadVideoToR2(c, file, userId); } else if (GIF_UPLOAD_ALLOWED && BSKY_GIF_MIME_TYPES.includes(fileType)) { // TODO: modify this in the future to transform the image to a webm // then push to uploadVideo return uploadVideoToR2(c, file, userId); } } return { success: false, error: "unable to push to R2" }; }; export const getAllFilesList = async (c: AllContext) => { const options: R2ListOptions = { limit: 1000, include: ["customMetadata"] }; const values: R2BucketObject[] = []; let hasHitEnd = false; while (!hasHitEnd) { const response = await c.env.R2.list(options); for (const file of response.objects) { values.push({ name: file.key, user: file.customMetadata?.user ?? null, date: file.uploaded }); } if (response.truncated) options.cursor = response.cursor; else hasHitEnd = true; } return values; };