// Say - TTS API with OpenAI (default) and Google Cloud support // Caches audio to Digital Ocean Spaces for efficiency const OpenAI = require("openai"); const tts = require("@google-cloud/text-to-speech"); const crypto = require("crypto"); const { S3Client, HeadObjectCommand, PutObjectCommand } = require("@aws-sdk/client-s3"); // OpenAI voice mapping // gpt-4o-mini-tts voices: alloy, ash, ballad, coral, echo, fable, nova, onyx, sage, shimmer, verse const OPENAI_VOICES = { male: ["onyx", "echo", "ash", "alloy"], female: ["nova", "shimmer", "fable", "coral"], neutral: ["alloy", "fable", "echo", "sage", "verse", "ballad"], }; // Initialize S3 client for Digital Ocean Spaces const s3 = new S3Client({ endpoint: `https://${process.env.ART_ENDPOINT}`, region: "us-east-1", // DO Spaces requires a region, but it's ignored credentials: { accessKeyId: process.env.ART_KEY, secretAccessKey: process.env.ART_SECRET, }, }); const BUCKET = process.env.ART_SPACE_NAME; const CDN_URL = "https://art.aesthetic.computer"; const CACHE_PREFIX = "tts-cache/"; // Generate cache key from provider + voice + text + instructions // Consented voice clones get their own subfolders so their utterance catalogs // stay distinct from stock voices. function getCacheKey(provider, voiceId, text, instructions) { const parts = `${provider}:${voiceId}:${text}${instructions ? `:${instructions}` : ""}`; const hash = crypto.createHash("sha256").update(parts).digest("hex"); const subfolder = provider === "jeffrey" ? "jeffrey/" : provider === "prutti" ? "pruttivox/" : ""; return `${CACHE_PREFIX}${subfolder}${hash}.mp3`; } // Check if cached audio exists, return CDN URL if so async function checkCache(key) { try { await s3.send(new HeadObjectCommand({ Bucket: BUCKET, Key: key })); return `${CDN_URL}/${key}`; } catch (err) { if (err.name === "NotFound" || err.$metadata?.httpStatusCode === 404) { return null; // Not cached } console.error("Cache check error:", err); return null; } } // Log every utterance event into the `sayings` MongoDB collection. // Modeled after the `moods` collection — ledger of timestamped events. // Failures here are swallowed so TTS responses never fail because of logging. async function recordSaying(entry) { let database; try { const { connect } = await import("../../backend/database.mjs"); database = await connect(); const collection = database.db.collection("sayings"); await collection.createIndex({ when: -1 }); await collection.createIndex({ provider: 1, when: -1 }); await collection.createIndex({ cacheKey: 1 }); await collection.insertOne({ ...entry, when: new Date() }); } catch (err) { console.error("⚠️ sayings log failed:", err?.message || err); } finally { if (database) { try { await database.disconnect(); } catch (_) {} } } } // Save audio to cache. Optional `metadata` is persisted as S3 user metadata // so we can browse utterances later (e.g. HeadObject → x-amz-meta-text). async function saveToCache(key, audioBuffer, metadata = {}) { try { // S3 user metadata must be ASCII and each header is usually capped // around 2 KB; trim text + encode non-ASCII defensively. const cleanMeta = {}; for (const [k, v] of Object.entries(metadata)) { if (v == null) continue; const str = String(v).slice(0, 1800); // Keep values ASCII-safe (S3 rejects high-unicode metadata headers). cleanMeta[k] = Buffer.from(str, "utf8").toString("ascii").replace(/[\r\n]/g, " "); } await s3.send(new PutObjectCommand({ Bucket: BUCKET, Key: key, Body: audioBuffer, ContentType: "audio/mpeg", ACL: "public-read", CacheControl: "public, max-age=31536000", // 1 year (audio doesn't change) Metadata: cleanMeta, })); console.log(`✅ Cached TTS: ${CDN_URL}/${key}`); return `${CDN_URL}/${key}`; } catch (err) { console.error("Cache write error:", err); return null; } } // Generate audio with OpenAI TTS // Uses gpt-4o-mini-tts when instructions are provided (supports emotional/style control), // falls back to tts-1 otherwise. async function generateOpenAI(text, gender, set, instructions) { const voiceList = OPENAI_VOICES[gender] || OPENAI_VOICES.neutral; const voice = voiceList[set % voiceList.length]; const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); const params = { model: instructions ? "gpt-4o-mini-tts" : "tts-1", voice: voice, input: text, response_format: "mp3", }; if (instructions) params.instructions = instructions; const mp3Response = await openai.audio.speech.create(params); return { buffer: Buffer.from(await mp3Response.arrayBuffer()), voiceId: `openai-${voice}`, }; } // ElevenLabs voice mapping (premade voice IDs) const ELEVEN_VOICES = { male: [ "SOYHLrjzK2X1ezoPC6cr", // Harry - Fierce Warrior "IKne3meq5aSn9XLyUdCD", // Charlie - Deep, Confident, Energetic "N2lVS1w4EtoT3dr4eOWO", // Callum - Husky Trickster "TX3LPaxmHKxFdv7VOQHJ", // Liam - Energetic ], female: [ "EXAVITQu4vr4xnSDxMaL", // Sarah "FGY2WhTYpPnrIDTdsKH5", // Laura "cgSgspJ2msm6clMCkdW9", // Jessica ], neutral: [ "SAz9YHcvj6GT2YYXdXww", // River - Relaxed, Neutral "cjVigY5qzO86Huf0OWal", // Eric "bIHbv24MWmeRgasZH58o", // Will ], }; // Generate audio with ElevenLabs TTS async function generateElevenLabs(text, gender, set, scream, withTimestamps = false) { const voiceList = ELEVEN_VOICES[gender] || ELEVEN_VOICES.neutral; const voiceId = voiceList[set % voiceList.length]; const voiceSettings = scream ? { stability: 0.1, similarity_boost: 0.7, style: 1.0, use_speaker_boost: true } : { stability: 0.5, similarity_boost: 0.75, style: 0.4, use_speaker_boost: true }; const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${voiceId}${withTimestamps ? "/with-timestamps" : ""}`, { method: "POST", headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: "eleven_multilingual_v2", voice_settings: voiceSettings, }), }); if (!response.ok) { const err = await response.text(); throw new Error(`ElevenLabs API error ${response.status}: ${err}`); } if (withTimestamps) { const json = await response.json(); return {buffer: Buffer.from(json.audio_base64, "base64"), voiceId: `eleven-${voiceId.slice(0, 8)}`, alignment: json.alignment, normalizedAlignment: json.normalized_alignment}; } return { buffer: Buffer.from(await response.arrayBuffer()), voiceId: `eleven-${voiceId.slice(0, 8)}`, }; } // Pruttivox uses Prutti's consented IVC. Arbitrary text is open to any // signed-in handle — the community lane in marketing/klokkentales/SCORE.md, // where Prutti hears every clip and can veto it. The anonymous chat lane // remains message-id-only in `netlify/functions/pruttivox.mjs`. async function generatePrutti(text) { const voiceId = process.env.PRUTTI_ELEVENLABS_VOICE_ID; if (!voiceId || !process.env.ELEVENLABS_API_KEY) { const error = new Error("Pruttivox is not configured."); error.statusCode = 503; throw error; } const response = await fetch( `https://api.elevenlabs.io/v1/text-to-speech/${encodeURIComponent(voiceId)}`, { method: "POST", headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: "eleven_multilingual_v2", voice_settings: { stability: 0.38, similarity_boost: 0.9, style: 0.48, use_speaker_boost: true, speed: 0.98, }, }), }, ); if (!response.ok) { const detail = (await response.text()).slice(0, 300); throw new Error(`ElevenLabs (Pruttivox) ${response.status}: ${detail}`); } return { buffer: Buffer.from(await response.arrayBuffer()), voiceId: "prutti-ivc", }; } async function authorizePruttivox(event) { const { authorize, handleFor } = await import("../../backend/authorization.mjs"); const user = await authorize(event.headers); if (!user?.sub) return null; // Any handle may send a text. A handle already implies a verified email // (handle.mjs won't mint one otherwise), and it's logged with the render, // so a clip can always be traced back to whoever asked for it. const handle = String(await handleFor(user.sub) || "").toLowerCase(); return handle ? `@${handle}` : null; } // ── Jeffrey: Professional Voice Clone (PVC) ────────────────────────── // Trained on multiple public lectures/talks by @jeffrey. Same voice // used in the LACMA 2026 grant pitch video. // Usage from the piece: `say:jeffrey hello world` const JEFFREY_VOICE_ID = "ZXoQQp5X0PKHGwyZpVIT"; // When `withTimestamps` is true, hits ElevenLabs `/with-timestamps` // endpoint and returns BOTH audio + per-character alignment. The // alignment is the source-of-truth replacement for whisper STT post- // processing; it is exact (no recognition, no formant distortion in the // signal yet) and free. async function generateJeffrey(text, scream, speed = 1.0, styleOverride = null, stabilityOverride = null, similarityOverride = null, withTimestamps = false) { // Calmer, more natural delivery than the premade "scream" preset. // Same knobs as the grant-video pipeline for homogeneity. // ElevenLabs voice_settings exposed (eleven_multilingual_v2): // speed 0.7-1.2 speech rate // style 0-1 emotional exaggeration (higher = more dramatic) // stability 0-1 lower = more expressive variation, higher = uniform // similarity 0-1 closeness to source clone (default 0.9) const baseStyle = scream ? 0.9 : 0.15; const baseStability = scream ? 0.2 : 0.65; const baseSimilarity = scream ? 0.85 : 0.9; const style = (styleOverride !== null) ? styleOverride : baseStyle; const stability = (stabilityOverride !== null) ? stabilityOverride : baseStability; const similarity = (similarityOverride !== null) ? similarityOverride : baseSimilarity; const voiceSettings = { stability, similarity_boost: similarity, style, use_speaker_boost: true, speed, }; const baseUrl = `https://api.elevenlabs.io/v1/text-to-speech/${JEFFREY_VOICE_ID}`; const url = withTimestamps ? `${baseUrl}/with-timestamps` : baseUrl; const response = await fetch(url, { method: "POST", headers: { "xi-api-key": process.env.ELEVENLABS_API_KEY, "Content-Type": "application/json", }, body: JSON.stringify({ text, model_id: "eleven_multilingual_v2", voice_settings: voiceSettings, }), }); if (!response.ok) { const err = await response.text(); throw new Error(`ElevenLabs (Jeffrey) API error ${response.status}: ${err}`); } if (withTimestamps) { // /with-timestamps returns JSON: { audio_base64, alignment, normalized_alignment } const json = await response.json(); return { buffer: Buffer.from(json.audio_base64, "base64"), voiceId: "jeffrey-pvc", alignment: json.alignment, normalizedAlignment: json.normalized_alignment, }; } return { buffer: Buffer.from(await response.arrayBuffer()), voiceId: "jeffrey-pvc", }; } // Generate audio with Google Cloud TTS async function generateGoogle(text, gender, set, isSSML) { // Fetch GCP key from URL const response = await fetch(process.env.GCP_TTS_KEY_URL); if (!response.ok) { throw new Error(`Failed to fetch GCP key: ${response.status}`); } const json = await response.json(); const gcpKey = json.GCP_TTS_KEY; const client = new tts.TextToSpeechClient({ credentials: { private_key: gcpKey.replace(/\\n/g, "\n"), client_email: process.env.GCP_EMAIL, }, }); // Get available voices const voices = (await client.listVoices({ languageCode: "en-US" }))[0].voices; const females = voices .filter((v) => v.ssmlGender === "FEMALE") .sort((a, b) => a.name.localeCompare(b.name)); const males = voices .filter((v) => v.ssmlGender === "MALE") .sort((a, b) => a.name.localeCompare(b.name)); let voice; const genderUpper = gender.toUpperCase(); if (genderUpper === "MALE") { voice = males[set % males.length]; } else if (genderUpper === "FEMALE") { voice = females[set % females.length]; } else { voice = males[1 % males.length]; } const ttsRequest = { voice: { languageCode: "en-US", ...voice }, audioConfig: { audioEncoding: "MP3" }, input: isSSML ? { ssml: text } : { text }, }; const [ttsResponse] = await client.synthesizeSpeech(ttsRequest); return { buffer: ttsResponse.audioContent, voiceId: `google-${voice.name}`, }; } exports.handler = async (event) => { const method = event.httpMethod; const headers = corsHeaders(event); if (method === "OPTIONS") { return { statusCode: 200, headers, body: JSON.stringify({ message: "Success!" }), }; } else if (method === "POST") { const body = JSON.parse(event.body); const utterance = body.from || "aesthetic.computer"; const set = parseInt(body.voice?.split(":")[1]) || 0; const gender = body.voice?.split(":")[0]?.toLowerCase() || "neutral"; // Provider: "jeffrey" (default PVC), "openai", "google", "eleven", // or handle-gated "prutti". // Can be set via body.provider; falls back to Jeffrey for parity // with the `say` piece default. const provider = body.provider || "jeffrey"; let requester = null; if (provider === "prutti") { if (typeof body.from !== "string" || !body.from.trim()) { return { statusCode: 400, headers, body: JSON.stringify({ message: "Pruttivox needs text to speak." }), }; } try { requester = await authorizePruttivox(event); } catch (error) { console.error("Pruttivox authorization failed:", error); return { statusCode: 503, headers, body: JSON.stringify({ message: "Pruttivox authorization is unavailable." }), }; } if (!requester) { return { statusCode: 403, headers, body: JSON.stringify({ message: "Pruttivox needs a signed-in handle." }), }; } if (utterance.length > 1200) { return { statusCode: 413, headers, body: JSON.stringify({ message: "Pruttivox chunks must be 1200 characters or fewer." }), }; } } // Instructions for gpt-4o-mini-tts emotional/style control (OpenAI only) const instructions = provider === "openai" ? (body.instructions || null) : null; // Scream mode for ElevenLabs (low stability, max style) const scream = body.scream === true; // Speed for ElevenLabs voice_settings.speed (0.7-1.2 range, // supported by eleven_multilingual_v2 + newer models). Default 1. const speed = (typeof body.speed === "number") ? Math.max(0.7, Math.min(1.2, body.speed)) : 1.0; // Style exaggeration for ElevenLabs voice_settings.style (0-1). // Higher = more emotional / dramatic delivery. null = use defaults. const styleOverride = (typeof body.style === "number") ? Math.max(0, Math.min(1, body.style)) : null; // Stability (0-1). Lower = more variable / expressive, higher = // more uniform / calm. null = use defaults. const stabilityOverride = (typeof body.stability === "number") ? Math.max(0, Math.min(1, body.stability)) : null; // Similarity boost (0-1). Higher = closer to original voice clone. // Default stays 0.9 — don't drop it without good reason. const similarityOverride = (typeof body.similarity === "number") ? Math.max(0, Math.min(1, body.similarity)) : null; // Cache bust: if true, skip cache lookup and regenerate const bustCache = body.bust === true; // Timestamps: when true, hit ElevenLabs `/with-timestamps` endpoint // (jeffrey provider only) and return JSON `{audio_base64, alignment}` // instead of raw mp3. Cache lookup is skipped because cached entries // are audio-only — cache write still happens (mp3 only) so future // non-timestamp callers benefit. Backward compatible: existing // recap/slab callers never set this flag and continue to receive // raw mp3 (302 redirect to CDN). const withTimestamps = body.withTimestamps === true || body.with_timestamps === true; // Check for SSML (only Google supports it) const isSSML = utterance.indexOf("") !== -1; // Strip SSML tags for OpenAI (it doesn't support them) let text = utterance; if (isSSML && provider === "openai") { text = text.replace(/<[^>]*>/g, "").trim(); } // Build voice identifier for cache key. Speed becomes part of the // cache key so a slow-render and a fast-render produce different // cached entries. const speedSuffix = (speed !== 1.0) ? `-spd${speed.toFixed(2)}` : ""; const styleSuffix = (styleOverride !== null) ? `-sty${styleOverride.toFixed(2)}` : ""; const stabSuffix = (stabilityOverride !== null) ? `-stb${stabilityOverride.toFixed(2)}` : ""; const simSuffix = (similarityOverride !== null) ? `-sim${similarityOverride.toFixed(2)}` : ""; const voiceSpec = `${provider}-${gender}-${set}${scream ? "-scream" : ""}${speedSuffix}${styleSuffix}${stabSuffix}${simSuffix}`; const cacheKey = getCacheKey(provider, voiceSpec, text, instructions); try { // Check cache first - return redirect to CDN if cached (unless bust=true). // When withTimestamps is set we always regenerate, because the cache // only stores the audio bytes — alignment must come fresh from the API. if (!bustCache && !withTimestamps) { const cachedUrl = await checkCache(cacheKey); if (cachedUrl) { console.log(`🎯 TTS cache hit: ${cachedUrl}`); await recordSaying({ text, provider, requester, requestId: body.requestId || null, chunkIndex: Number.isInteger(body.chunkIndex) ? body.chunkIndex : null, chunkCount: Number.isInteger(body.chunkCount) ? body.chunkCount : null, voice: null, // unknown on cache hit; cacheKey ties it to the original voiceSpec, scream, instructions, cacheKey, url: cachedUrl, cached: true, }); return { statusCode: 302, headers: { ...headers, Location: cachedUrl, "Cache-Control": "public, max-age=86400", }, body: "", }; } } else { console.log(`🧹 Cache bust requested for: ${text.substring(0, 50)}...`); } // Cache miss (or bust) - generate with selected provider console.log(`🔄 TTS ${bustCache ? "regenerating" : "cache miss"} (${provider}): ${text.substring(0, 50)}...`); let result; if (provider === "google") { result = await generateGoogle(text, gender, set, isSSML); } else if (provider === "eleven") { result = await generateElevenLabs(text, gender, set, scream, withTimestamps); } else if (provider === "prutti") { result = await generatePrutti(text); } else if (provider === "jeffrey") { result = await generateJeffrey(text, scream, speed, styleOverride, stabilityOverride, similarityOverride, withTimestamps); } else { result = await generateOpenAI(text, gender, set, instructions); } const { buffer: audioBuffer, voiceId, alignment, normalizedAlignment } = result; if (!audioBuffer || audioBuffer.length === 0) { return { statusCode: 500, headers, body: JSON.stringify({ message: "Failed to generate audio." }), }; } console.log(`🗣️ Generated with ${provider}: ${voiceId}`); // Cache for next time. Attach the original text + voice as S3 metadata // so individual objects are self-describing when you browse them. const cdnUrl = await saveToCache(cacheKey, audioBuffer, { text, provider, voice: voiceId, scream: scream ? "1" : "0", ts: new Date().toISOString(), }); // ── Timestamped response — JSON with audio + alignment ──────── // Returned only when caller opted in. Default callers (recap, // slab, the `say` piece) never see this branch and keep getting // a 302 → CDN raw-mp3. if (withTimestamps && alignment) { await recordSaying({ text, provider, requester, requestId: body.requestId || null, chunkIndex: Number.isInteger(body.chunkIndex) ? body.chunkIndex : null, chunkCount: Number.isInteger(body.chunkCount) ? body.chunkCount : null, voice: voiceId, voiceSpec, scream, instructions, cacheKey, url: cdnUrl, cached: false, withTimestamps: true, }); return { statusCode: 200, headers: { ...headers, "Content-Type": "application/json", }, body: JSON.stringify({ audio: audioBuffer.toString("base64"), alignment, normalizedAlignment: normalizedAlignment || null, url: cdnUrl, voice: voiceId, }), }; } if (cdnUrl) { await recordSaying({ text, provider, requester, requestId: body.requestId || null, chunkIndex: Number.isInteger(body.chunkIndex) ? body.chunkIndex : null, chunkCount: Number.isInteger(body.chunkCount) ? body.chunkCount : null, voice: voiceId, voiceSpec, scream, instructions, cacheKey, url: cdnUrl, cached: false, }); return { statusCode: 302, headers: { ...headers, Location: cdnUrl, }, body: "", }; } // Fallback: return audio directly if caching failed return { statusCode: 200, headers: { ...headers, "Content-Disposition": 'inline; filename="response.mp3"', "Content-Type": "audio/mpeg", }, body: audioBuffer.toString("base64"), isBase64Encoded: true, }; } catch (error) { console.error("TTS generation failed:", error); return { statusCode: error.statusCode || 500, headers, body: JSON.stringify({ message: "An error has occurred.", error: error.message }), }; } } else { return { statusCode: 405, headers, body: JSON.stringify({ message: "Method Not Allowed" }), }; } }; function corsHeaders(event) { const dev = process.env.CONTEXT === "dev"; const production = !dev; let allowedOrigin = production ? "https://aesthetic.computer" : "*"; if (event.headers.origin === "null") allowedOrigin = "*"; return { "Access-Control-Allow-Methods": "GET,OPTIONS,PATCH,DELETE,POST,PUT", "Access-Control-Allow-Origin": allowedOrigin, "Access-Control-Allow-Credentials": true, "Access-Control-Allow-Headers": "X-CSRF-Token, X-Requested-With, Accept, Accept-Version, Content-Length, Content-MD5, Content-Type, Date, X-Api-Version, Authorization", }; }