Something went wrong. Try again.
Monorepo for Aesthetic.Computer aesthetic.computer
Something went wrong. Try again.
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559// Say - TTS API with OpenAI (default) and Google Cloud support// Caches audio to Digital Ocean Spaces for efficiency
const OpenAI = require("openai");const tts = require("@google-cloud/text-to-speech");const crypto = require("crypto");const { S3Client, HeadObjectCommand, PutObjectCommand } = require("@aws-sdk/client-s3");
// OpenAI voice mapping// gpt-4o-mini-tts voices: alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verseconst OPENAI_VOICES = { male: ["onyx", "echo", "ash", "alloy"], female: ["nova", "shimmer", "fable", "coral"], neutral: ["alloy", "fable", "echo", "sage", "verse", "ballad"],};
// Initialize S3 client for Digital Ocean Spacesconst s3 = new S3Client({ endpoint: `https://${process.env.ART_ENDPOINT}`, region: "us-east-1", // DO Spaces requires a region, but it's ignored credentials: { accessKeyId: process.env.ART_KEY, secretAccessKey: process.env.ART_SECRET, },});
const BUCKET = process.env.ART_SPACE_NAME;const CDN_URL = "https://art.aesthetic.computer";const CACHE_PREFIX = "tts-cache/";
// Generate cache key from provider + voice + text + instructions// Custom voice clones (jeffrey) get their own subfolder so all utterances// from that voice are easy to list / browse on the CDN.function getCacheKey(provider, voiceId, text, instructions) { const parts = `${provider}:${voiceId}:${text}${instructions ? `:${instructions}` : ""}`; const hash = crypto.createHash("sha256").update(parts).digest("hex"); const subfolder = provider === "jeffrey" ? "jeffrey/" : ""; return `${CACHE_PREFIX}${subfolder}${hash}.mp3`;}
// Check if cached audio exists, return CDN URL if soasync function checkCache(key) { try { await s3.send(new HeadObjectCommand({ Bucket: BUCKET, Key: key })); return `${CDN_URL}/${key}`; } catch (err) { if (err.name === "NotFound" || err.$metadata?.httpStatusCode === 404) { return null; // Not cached } console.error("Cache check error:", err); return null; }}
// Log every utterance event into the `sayings` MongoDB collection.// Modeled after the `moods` collection — ledger of timestamped events.// Failures here are swallowed so TTS responses never fail because of logging.async function recordSaying(entry) { let database; try { const { connect } = await import("../../backend/database.mjs"); database = await connect(); const collection = database.db.collection("sayings"); await collection.createIndex({ when: -1 }); await collection.createIndex({ provider: 1, when: -1 }); await collection.createIndex({ cacheKey: 1 }); await collection.insertOne({ ...entry, when: new Date() }); } catch (err) { console.error("⚠️ sayings log failed:", err?.message || err); } finally { if (database) { try { await database.disconnect(); } catch (_) {} } }}
// Save audio to cache. Optional `metadata` is persisted as S3 user metadata// so we can browse utterances later (e.g. HeadObject → x-amz-meta-text).async function saveToCache(key, audioBuffer, metadata = {}) { try { // S3 user metadata must be ASCII and each header is usually capped // around 2 KB; trim text + encode non-ASCII defensively. const cleanMeta = {}; for (const [k, v] of Object.entries(metadata)) { if (v == null) continue; const str = String(v).slice(0, 1800); // Keep values ASCII-safe (S3 rejects high-unicode metadata headers). cleanMeta[k] = Buffer.from(str, "utf8").toString("ascii").replace(/[\r\n]/g, " "); }
await s3.send(new PutObjectCommand({ Bucket: BUCKET, Key: key, Body: audioBuffer, ContentType: "audio/mpeg", ACL: "public-read", CacheControl: "public, max-age=31536000", // 1 year (audio doesn't change) Metadata: cleanMeta, })); console.log(`✅ Cached TTS: ${CDN_URL}/${key}`); return `${CDN_URL}/${key}`; } catch (err) { console.error("Cache write error:", err); return null; }}
// Generate audio with OpenAI TTS// Uses gpt-4o-mini-tts when instructions are provided (supports emotional/style control),// falls back to tts-1 otherwise.async function generateOpenAI(text, gender, set, instructions) { const voiceList = OPENAI_VOICES[gender] || OPENAI_VOICES.neutral; const voice = voiceList[set % voiceList.length];
const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY });
const params = { model: instructions ? "gpt-4o-mini-tts" : "tts-1", voice: voice, input: text, response_format: "mp3", };
if (instructions) params.instructions = instructions;
const mp3Response = await openai.audio.speech.create(params);
return { buffer: Buffer.from(await mp3Response.arrayBuffer()), voiceId: `openai-${voice}`, };}
// ElevenLabs voice mapping (premade voice IDs)const ELEVEN_VOICES = { male: [ "SOYHLrjzK2X1ezoPC6cr", // Harry - Fierce Warrior "IKne3meq5aSn9XLyUdCD", // Charlie - Deep, Confident, Energetic "N2lVS1w4EtoT3dr4eOWO", // Callum - Husky Trickster "TX3LPaxmHKxFdv7VOQHJ", // Liam - Energetic ], female: [ "EXAVITQu4vr4xnSDxMaL", // Sarah "FGY2WhTYpPnrIDTdsKH5", // Laura "cgSgspJ2msm6clMCkdW9", // Jessica ], neutral: [ "SAz9YHcvj6GT2YYXdXww", // River - Relaxed, Neutral "cjVigY5qzO86Huf0OWal", // Eric "bIHbv24MWmeRgasZH58o", // Will ],};
// Generate audio with ElevenLabs TTSasync function generateElevenLabs(text, gender, set, scream) { const voiceList = ELEVEN_VOICES[gender] || ELEVEN_VOICES.neutral; const voiceId = voiceList[set % voiceList.length];
const voiceSettings = scream ? { stability: 0.1, similarity_boost: 0.7, style: 1.0, use_speaker_boost: true } : { stability: 0.5, similarity_boost: 0.75, style: 0.4, use_speaker_boost: true };
const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`, { method: "POST", headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: "eleven_multilingual_v2", voice_settings: voiceSettings, }), });
if (!response.ok) { const err = await response.text(); throw new Error(`ElevenLabs API error ${response.status}: ${err}`); }
return { buffer: Buffer.from(await response.arrayBuffer()), voiceId: `eleven-${voiceId.slice(0, 8)}`, };}
// ── Jeffrey: Professional Voice Clone (PVC) ──────────────────────────// Trained on multiple public lectures/talks by @jeffrey. Same voice// used in the LACMA 2026 grant pitch video.// Usage from the piece: `say:jeffrey hello world`const JEFFREY_VOICE_ID = "ZXoQQp5X0PKHGwyZpVIT";
// When `withTimestamps` is true, hits ElevenLabs `/with-timestamps`// endpoint and returns BOTH audio + per-character alignment. The// alignment is the source-of-truth replacement for whisper STT post-// processing; it is exact (no recognition, no formant distortion in the// signal yet) and free.async function generateJeffrey(text, scream, speed = 1.0, styleOverride = null, stabilityOverride = null, similarityOverride = null, withTimestamps = false) { // Calmer, more natural delivery than the premade "scream" preset. // Same knobs as the grant-video pipeline for homogeneity. // ElevenLabs voice_settings exposed (eleven_multilingual_v2): // speed 0.7-1.2 speech rate // style 0-1 emotional exaggeration (higher = more dramatic) // stability 0-1 lower = more expressive variation, higher = uniform // similarity 0-1 closeness to source clone (default 0.9) const baseStyle = scream ? 0.9 : 0.15; const baseStability = scream ? 0.2 : 0.65; const baseSimilarity = scream ? 0.85 : 0.9; const style = (styleOverride !== null) ? styleOverride : baseStyle; const stability = (stabilityOverride !== null) ? stabilityOverride : baseStability; const similarity = (similarityOverride !== null) ? similarityOverride : baseSimilarity; const voiceSettings = { stability, similarity_boost: similarity, style, use_speaker_boost: true, speed, };
const baseUrl = `https://api.elevenlabs.io/v1/text-to-speech/${JEFFREY_VOICE_ID}`; const url = withTimestamps ? `${baseUrl}/with-timestamps` : baseUrl;
const response = await fetch(url, { method: "POST", headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: "eleven_multilingual_v2", voice_settings: voiceSettings, }), });
if (!response.ok) { const err = await response.text(); throw new Error(`ElevenLabs (Jeffrey) API error ${response.status}: ${err}`); }
if (withTimestamps) { // /with-timestamps returns JSON: { audio_base64, alignment, normalized_alignment } const json = await response.json(); return { buffer: Buffer.from(json.audio_base64, "base64"), voiceId: "jeffrey-pvc", alignment: json.alignment, normalizedAlignment: json.normalized_alignment, }; }
return { buffer: Buffer.from(await response.arrayBuffer()), voiceId: "jeffrey-pvc", };}
// Generate audio with Google Cloud TTSasync function generateGoogle(text, gender, set, isSSML) { // Fetch GCP key from URL const response = await fetch(process.env.GCP_TTS_KEY_URL); if (!response.ok) { throw new Error(`Failed to fetch GCP key: ${response.status}`); } const json = await response.json(); const gcpKey = json.GCP_TTS_KEY;
const client = new tts.TextToSpeechClient({ credentials: { private_key: gcpKey.replace(/\\n/g, "\n"), client_email: process.env.GCP_EMAIL, }, });
// Get available voices const voices = (await client.listVoices({ languageCode: "en-US" }))[0].voices;
const females = voices .filter((v) => v.ssmlGender === "FEMALE") .sort((a, b) => a.name.localeCompare(b.name));
const males = voices .filter((v) => v.ssmlGender === "MALE") .sort((a, b) => a.name.localeCompare(b.name));
let voice; const genderUpper = gender.toUpperCase(); if (genderUpper === "MALE") { voice = males[set % males.length]; } else if (genderUpper === "FEMALE") { voice = females[set % females.length]; } else { voice = males[1 % males.length]; }
const ttsRequest = { voice: { languageCode: "en-US", ...voice }, audioConfig: { audioEncoding: "MP3" }, input: isSSML ? { ssml: text } : { text }, };
const [ttsResponse] = await client.synthesizeSpeech(ttsRequest);
return { buffer: ttsResponse.audioContent, voiceId: `google-${voice.name}`, };}
exports.handler = async (event) => { const method = event.httpMethod; const headers = corsHeaders(event);
if (method === "OPTIONS") { return { statusCode: 200, headers, body: JSON.stringify({ message: "Success!" }), }; } else if (method === "POST") { const body = JSON.parse(event.body);
const utterance = body.from || "aesthetic.computer"; const set = parseInt(body.voice?.split(":")[1]) || 0; const gender = body.voice?.split(":")[0]?.toLowerCase() || "neutral";
// Provider: "jeffrey" (default PVC), "openai", "google", "eleven" // Can be set via body.provider; falls back to Jeffrey for parity // with the `say` piece default. const provider = body.provider || "jeffrey";
// Instructions for gpt-4o-mini-tts emotional/style control (OpenAI only) const instructions = provider === "openai" ? (body.instructions || null) : null;
// Scream mode for ElevenLabs (low stability, max style) const scream = body.scream === true;
// Speed for ElevenLabs voice_settings.speed (0.7-1.2 range, // supported by eleven_multilingual_v2 + newer models). Default 1. const speed = (typeof body.speed === "number") ? Math.max(0.7, Math.min(1.2, body.speed)) : 1.0; // Style exaggeration for ElevenLabs voice_settings.style (0-1). // Higher = more emotional / dramatic delivery. null = use defaults. const styleOverride = (typeof body.style === "number") ? Math.max(0, Math.min(1, body.style)) : null; // Stability (0-1). Lower = more variable / expressive, higher = // more uniform / calm. null = use defaults. const stabilityOverride = (typeof body.stability === "number") ? Math.max(0, Math.min(1, body.stability)) : null; // Similarity boost (0-1). Higher = closer to original voice clone. // Default stays 0.9 — don't drop it without good reason. const similarityOverride = (typeof body.similarity === "number") ? Math.max(0, Math.min(1, body.similarity)) : null;
// Cache bust: if true, skip cache lookup and regenerate const bustCache = body.bust === true;
// Timestamps: when true, hit ElevenLabs `/with-timestamps` endpoint // (jeffrey provider only) and return JSON `{audio_base64, alignment}` // instead of raw mp3. Cache lookup is skipped because cached entries // are audio-only — cache write still happens (mp3 only) so future // non-timestamp callers benefit. Backward compatible: existing // recap/slab callers never set this flag and continue to receive // raw mp3 (302 redirect to CDN). const withTimestamps = body.withTimestamps === true || body.with_timestamps === true;
// Check for SSML (only Google supports it) const isSSML = utterance.indexOf("<speak>") !== -1;
// Strip SSML tags for OpenAI (it doesn't support them) let text = utterance; if (isSSML && provider === "openai") { text = text.replace(/<[^>]*>/g, "").trim(); }
// Build voice identifier for cache key. Speed becomes part of the // cache key so a slow-render and a fast-render produce different // cached entries. const speedSuffix = (speed !== 1.0) ? `-spd${speed.toFixed(2)}` : ""; const styleSuffix = (styleOverride !== null) ? `-sty${styleOverride.toFixed(2)}` : ""; const stabSuffix = (stabilityOverride !== null) ? `-stb${stabilityOverride.toFixed(2)}` : ""; const simSuffix = (similarityOverride !== null) ? `-sim${similarityOverride.toFixed(2)}` : ""; const voiceSpec = `${provider}-${gender}-${set}${scream ? "-scream" : ""}${speedSuffix}${styleSuffix}${stabSuffix}${simSuffix}`; const cacheKey = getCacheKey(provider, voiceSpec, text, instructions);
try { // Check cache first - return redirect to CDN if cached (unless bust=true). // When withTimestamps is set we always regenerate, because the cache // only stores the audio bytes — alignment must come fresh from the API. if (!bustCache && !withTimestamps) { const cachedUrl = await checkCache(cacheKey);
if (cachedUrl) { console.log(`🎯 TTS cache hit: ${cachedUrl}`); await recordSaying({ text, provider, voice: null, // unknown on cache hit; cacheKey ties it to the original voiceSpec, scream, instructions, cacheKey, url: cachedUrl, cached: true, }); return { statusCode: 302, headers: { ...headers, Location: cachedUrl, "Cache-Control": "public, max-age=86400", }, body: "", }; } } else { console.log(`🧹 Cache bust requested for: ${text.substring(0, 50)}...`); }
// Cache miss (or bust) - generate with selected provider console.log(`🔄 TTS ${bustCache ? "regenerating" : "cache miss"} (${provider}): ${text.substring(0, 50)}...`);
let result; if (provider === "google") { result = await generateGoogle(text, gender, set, isSSML); } else if (provider === "eleven") { result = await generateElevenLabs(text, gender, set, scream); } else if (provider === "jeffrey") { result = await generateJeffrey(text, scream, speed, styleOverride, stabilityOverride, similarityOverride, withTimestamps); } else { result = await generateOpenAI(text, gender, set, instructions); }
const { buffer: audioBuffer, voiceId, alignment, normalizedAlignment } = result;
if (!audioBuffer || audioBuffer.length === 0) { return { statusCode: 500, headers, body: JSON.stringify({ message: "Failed to generate audio." }), }; }
console.log(`🗣️ Generated with ${provider}: ${voiceId}`);
// Cache for next time. Attach the original text + voice as S3 metadata // so individual objects are self-describing when you browse them. const cdnUrl = await saveToCache(cacheKey, audioBuffer, { text, provider, voice: voiceId, scream: scream ? "1" : "0", ts: new Date().toISOString(), });
// ── Timestamped response — JSON with audio + alignment ──────── // Returned only when caller opted in. Default callers (recap, // slab, the `say` piece) never see this branch and keep getting // a 302 → CDN raw-mp3. if (withTimestamps && alignment) { await recordSaying({ text, provider, voice: voiceId, voiceSpec, scream, instructions, cacheKey, url: cdnUrl, cached: false, withTimestamps: true, }); return { statusCode: 200, headers: { ...headers, "Content-Type": "application/json", }, body: JSON.stringify({ audio: audioBuffer.toString("base64"), alignment, normalizedAlignment: normalizedAlignment || null, url: cdnUrl, voice: voiceId, }), }; }
if (cdnUrl) { await recordSaying({ text, provider, voice: voiceId, voiceSpec, scream, instructions, cacheKey, url: cdnUrl, cached: false, }); return { statusCode: 302, headers: { ...headers, Location: cdnUrl, }, body: "", }; }
// Fallback: return audio directly if caching failed return { statusCode: 200, headers: { ...headers, "Content-Disposition": 'inline; filename="response.mp3"', "Content-Type": "audio/mpeg", }, body: audioBuffer.toString("base64"), isBase64Encoded: true, }; } catch (error) { console.error("TTS generation failed:", error); return { statusCode: 500, headers, body: JSON.stringify({ message: "An error has occurred.", error: error.message }), }; } } else { return { statusCode: 405, headers, body: JSON.stringify({ message: "Method Not Allowed" }), }; }};
function corsHeaders(event) { const dev = process.env.CONTEXT === "dev"; const production = !dev; let allowedOrigin = production ? "https://aesthetic.computer" : "*"; if (event.headers.origin === "null") allowedOrigin = "*";
return { "Access-Control-Allow-Methods": "GET,OPTIONS,PATCH,DELETE,POST,PUT", "Access-Control-Allow-Origin": allowedOrigin, "Access-Control-Allow-Credentials": true, "Access-Control-Allow-Headers": "X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version", };}