/** * TTS (Text-to-Speech) Provider Implementation * * Factory pattern for routing TTS requests to appropriate provider implementations. * Follows the same architecture as lib/ai/providers.ts for consistency. * * Currently Supported Providers: * - OpenAI TTS: https://platform.openai.com/docs/guides/text-to-speech * - Azure TTS: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/text-to-speech * - GLM TTS: https://docs.bigmodel.cn/cn/guide/models/sound-and-video/glm-tts * - Qwen TTS: https://bailian.console.aliyun.com/ * - MiniMax TTS: https://platform.minimaxi.com/docs/api-reference/speech-t2a-http * - Doubao TTS: https://www.volcengine.com/docs/6561/1257543 * - ElevenLabs TTS: https://elevenlabs.io/docs/api-reference/text-to-speech/convert * - Google Gemini TTS: https://ai.google.dev/gemini-api/docs/speech-generation * - Browser Native: Web Speech API (client-side only) * * HOW TO ADD A NEW PROVIDER: * * 1. Add provider ID to TTSProviderId in lib/audio/types.ts * Example: | 'elevenlabs-tts' * * 2. Add provider configuration to lib/audio/constants.ts * Example: * 'elevenlabs-tts': { * id: 'elevenlabs-tts', * name: 'ElevenLabs', * requiresApiKey: true, * defaultBaseUrl: 'https://api.elevenlabs.io/v1', * icon: '/logos/elevenlabs.svg', * voices: [...], * supportedFormats: ['mp3', 'pcm'], * speedRange: { min: 0.5, max: 2.0, default: 1.0 } * } * * 3. Implement provider function in this file * Pattern: async function generateXxxTTS(config, text): Promise * - Validate config and build API request * - Handle API authentication (apiKey, headers) * - Convert provider-specific parameters (voice, speed, format) * - Return { audio: Uint8Array, format: string } * * Example: * async function generateElevenLabsTTS( * config: TTSModelConfig, * text: string * ): Promise { * const baseUrl = config.baseUrl || TTS_PROVIDERS['elevenlabs-tts'].defaultBaseUrl; * * const response = await fetch(`${baseUrl}/text-to-speech/${config.voice}`, { * method: 'POST', * headers: { * 'xi-api-key': config.apiKey!, * 'Content-Type': 'application/json', * }, * body: JSON.stringify({ * text, * model_id: 'eleven_multilingual_v2', * voice_settings: { * stability: 0.5, * similarity_boost: 0.75, * } * }), * }); * * if (!response.ok) { * throw new Error(`ElevenLabs TTS API error: ${response.statusText}`); * } * * const arrayBuffer = await response.arrayBuffer(); * return { * audio: new Uint8Array(arrayBuffer), * format: 'mp3', * }; * } * * 4. Add case to generateTTS() switch statement * case 'elevenlabs-tts': * return await generateElevenLabsTTS(config, text); * * 5. Add i18n translations in lib/i18n.ts * providerElevenLabsTTS: { zh: 'ElevenLabs TTS', en: 'ElevenLabs TTS' } * * Error Handling Patterns: * - Always validate API key if requiresApiKey is true * - Throw descriptive errors for API failures * - Include response.statusText or error messages from API * - For client-only providers (browser-native), throw error directing to client-side usage * * API Call Patterns: * - Direct API: Use fetch with appropriate headers and body format (recommended for better encoding support) * - SSML: For Azure-like providers requiring SSML markup * - URL-based: For providers returning audio URL (download in second step) */ import type { TTSModelConfig } from './types'; import { isCustomTTSProvider } from './types'; import { isQwenCloneVoice, resolveTTSModelForVoice, TTS_PROVIDERS, DEFAULT_TTS_VOICES, } from './constants'; import { downloadAudio, QwenVoiceCloneError, synthesizeQwenVoiceClone } from './qwen-voice-clone'; import { evictQwenVoiceRegistrationMemo } from './qwen-voice-clone-registration'; import { splitConcatenatedJsonObjects } from './json-stream'; import { VOXCPM_VLLM_MODEL_ID, VOXCPM_AUTO_VOICE_ID, normalizeVoxCPMBackend, type VoxCPMProviderOptions, } from './voxcpm'; import { createLogger } from '@/lib/logger'; import { audioEndpointPolicy, audioProviderFetch, type AudioEndpointTarget, } from '@/lib/server/audio-provider-fetch'; import { ProviderQuotaExhaustedError } from '@/lib/server/provider-quota'; import { appAttributionHeaders } from '@/lib/config/app-attribution'; import { pcmS16leMonoToWav } from './pcm-wav'; const log = createLogger('TTSProviders'); /** * Every server-side provider request goes through the strict redirect + * pinned-DNS helper under the address policy the route resolved: strict public * for a client BYOK URL, local networks allowed for a server-configured * provider, the operator policy for an unmanaged provider's default endpoint. */ function ttsFetch( target: AudioEndpointTarget | undefined, url: string, init: RequestInit, ): Promise { return audioProviderFetch(url, init, audioEndpointPolicy(target)); } /** * Result of TTS generation */ export interface TTSGenerationResult { audio: Uint8Array; format: string; } /** * Thrown when a TTS provider returns a rate-limit / concurrency-quota error. * Allows downstream consumers to distinguish rate-limit errors from other TTS failures. * * TODO: The API route currently catches all errors uniformly as GENERATION_FAILED. * This class enables future retry/backoff logic without changing the throw sites. */ export class TTSRateLimitError extends Error { /** Milliseconds from a `Retry-After` header, when the provider sent one. */ readonly retryAfterMs?: number; /** * Classifier sentinel, not the provider's HTTP status. Business-level rate * limits (MiniMax HTTP 200 + code 1002, Doubao business codes) still carry * 429 so `runFailureCode` and route retry map this class to `RATE_LIMITED` * and a retryable status. A quota error attached as `cause` still wins. */ readonly statusCode = 429; constructor( public readonly provider: string, message: string, retryAfterMs?: number, options?: ErrorOptions, ) { super(message, options); this.name = 'TTSRateLimitError'; if (retryAfterMs !== undefined && Number.isFinite(retryAfterMs) && retryAfterMs > 0) { this.retryAfterMs = retryAfterMs; } } } /** Neutral identity for ordinary Qwen TTS failures (never VC-branded). */ export class QwenTTSError extends Error { readonly code = 'QWEN_TTS_ERROR'; readonly httpStatus: number; constructor(message: string, httpStatus = 502) { super(message); this.name = 'QwenTTSError'; this.httpStatus = httpStatus; } } /** * Thrown when a TTS provider responds with HTTP 200 but returns a non-audio body * (such as HTML error page, web front-end response, or JSON without an audioUrl). * Prevents non-audio bytes from being stored, referenced, or billed. */ export class TTSInvalidResponseError extends Error { readonly code = 'TTS_INVALID_RESPONSE'; readonly httpStatus: number; constructor( public readonly provider: string, message: string, httpStatus = 502, ) { super(message); this.name = 'TTSInvalidResponseError'; this.httpStatus = httpStatus; } } /** * Per-request bound for one TTS provider call, ported from the reference * runtime's TTS bounds (30s request timeouts on the managed TTS clients). * Overridable via `TTS_REQUEST_TIMEOUT_MS` (ms) for deployments with slower * upstreams; a hung provider fails the call with this error instead of * wedging the session. */ const DEFAULT_TTS_REQUEST_TIMEOUT_MS = 30_000; function ttsRequestTimeoutMs(): number { const raw = process.env.TTS_REQUEST_TIMEOUT_MS?.trim(); const parsed = raw ? Number(raw) : NaN; return Number.isFinite(parsed) && parsed > 0 ? parsed : DEFAULT_TTS_REQUEST_TIMEOUT_MS; } /** * Thrown when a single TTS provider request exceeds {@link ttsRequestTimeoutMs}. * Distinct from `AbortSignal`-driven cancellation (session cancel): a timeout * is a provider failure the caller may retry. */ export class TTSRequestTimeoutError extends Error { constructor( public readonly provider: string, message: string, ) { super(message); this.name = 'TTSRequestTimeoutError'; } } /** Combine the caller's cancel signal with the per-request timeout. */ function ttsRequestSignal(callerSignal?: AbortSignal): AbortSignal { const timeout = AbortSignal.timeout(ttsRequestTimeoutMs()); return callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout; } /** True when `signal` aborted because the per-request timeout fired. */ function isTimeoutSignal(signal: AbortSignal): boolean { return ( signal.aborted && signal.reason instanceof DOMException && signal.reason.name === 'TimeoutError' ); } /** * `Retry-After` as milliseconds. Accepts delta-seconds or an HTTP-date. * Undefined when the header is missing or not a delay. */ function parseRetryAfterMs( header: string | null | undefined, now = Date.now(), ): number | undefined { if (!header) return undefined; const trimmed = header.trim(); if (!trimmed) return undefined; if (/^\d+(?:\.\d+)?$/.test(trimmed)) return Math.max(0, Number(trimmed) * 1000); const dateMs = Date.parse(trimmed); if (Number.isNaN(dateMs)) return undefined; return Math.max(0, dateMs - now); } /** * Map an upstream HTTP 429 to a typed {@link TTSRateLimitError} so the API route * can surface it as 429 instead of a generic 500. Call right after an * `!response.ok` check, before building the provider-specific error message. * Pass the `Retry-After` header when the response has one. */ export function throwIfTtsRateLimited( provider: string, status: number, retryAfterHeader?: string | null, options?: ErrorOptions, ): void { if (status === 429) { throw new TTSRateLimitError( provider, `${provider} TTS rate limit exceeded (HTTP 429)`, parseRetryAfterMs(retryAfterHeader), options, ); } } /** * Generate speech using specified TTS provider * * Every provider request is created with a signal that combines the caller's * cancel signal (`config.signal`) with the per-request timeout, so a session * cancel aborts the in-flight fetch within seconds and a hung provider fails * with {@link TTSRequestTimeoutError} instead of hanging forever. */ export async function generateTTS( config: TTSModelConfig, text: string, ): Promise { const provider = TTS_PROVIDERS[config.providerId as keyof typeof TTS_PROVIDERS]; // Validate API key if required (only for built-in providers with known config) if (provider?.requiresApiKey && !config.apiKey) { throw new Error(`API key required for TTS provider: ${config.providerId}`); } const signal = ttsRequestSignal(config.signal); try { switch (config.providerId) { case 'openai-tts': return await generateOpenAITTS(config, text, signal); case 'azure-tts': return await generateAzureTTS(config, text, signal); case 'glm-tts': return await generateGLMTTS(config, text, signal); case 'qwen-tts': return await generateQwenTTS(config, text, signal); case 'voxcpm-tts': return await generateVoxCPMTTS(config, text, signal); case 'minimax-tts': return await generateMiniMaxTTS(config, text, signal); case 'xiaomi-tts': return await generateXiaomiTTS(config, text, signal); case 'doubao-tts': return await generateDoubaoTTS(config, text, signal); case 'elevenlabs-tts': return await generateElevenLabsTTS(config, text, signal); case 'google-tts': return await generateGoogleTTS(config, text, signal); case 'lemonade-tts': return await generateLemonadeTTS(config, text, signal); case 'browser-native-tts': throw new Error( 'Browser Native TTS must be handled client-side using Web Speech API. This provider cannot be used on the server.', ); default: if (isCustomTTSProvider(config.providerId)) { return await generateOpenAITTS(config, text, signal); } throw new Error(`Unsupported TTS provider: ${config.providerId}`); } } catch (error) { // A caller cancel must propagate as-is so the enclosing run treats it as an // interruption, not a provider failure. if (config.signal?.aborted) throw error; if (isTimeoutSignal(signal)) { throw new TTSRequestTimeoutError( config.providerId, `TTS request timed out after ${ttsRequestTimeoutMs()}ms (provider ${config.providerId}) — the provider did not respond. Do not retry with the same input without splitting long narration text or increasing TTS_REQUEST_TIMEOUT_MS.`, ); } throw error; } } /** * OpenAI TTS implementation (direct API call with explicit UTF-8 encoding) */ async function generateOpenAITTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['openai-tts'].defaultBaseUrl; // Use gpt-4o-mini-tts for best quality and intelligent realtime applications const response = await ttsFetch(config, `${baseUrl}/audio/speech`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: config.modelId || 'gpt-4o-mini-tts', input: text, voice: config.voice, speed: config.speed || 1.0, // Ask for a container the browser can decode. OpenAI defaults to mp3, but // this same function serves every custom OpenAI-compatible provider and // their defaults differ — OpenRouter's /audio/speech defaults to raw // `pcm`, which arrives headerless, gets labelled mp3 below, and fails in // the client with "no supported source was found". Naming the format // removes the guess. Providers that ignore the field are unaffected. response_format: 'mp3', }), signal, }); if (!response.ok) { const error = await response.json().catch(() => ({ error: response.statusText })); // Retain the response without changing the HTTP status used by route retries. const options = { cause: { data: error } }; throwIfTtsRateLimited('OpenAI', response.status, response.headers?.get('retry-after'), options); throw new Error( `OpenAI TTS API error: ${error.error?.message || response.statusText}`, options, ); } return await validateTTSAudioResponse(response, 'OpenAI'); } /** * Lemonade TTS implementation (OpenAI-compatible /v1/audio/speech). */ async function generateLemonadeTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = (config.baseUrl || TTS_PROVIDERS['lemonade-tts'].defaultBaseUrl || '').replace( /\/$/, '', ); const modelId = config.modelId || TTS_PROVIDERS['lemonade-tts'].defaultModelId; const voice = config.voice || 'af_heart'; const response = await ttsFetch(config, `${baseUrl}/audio/speech`, { method: 'POST', headers: { 'Content-Type': 'application/json; charset=utf-8', ...getBackendAuthHeaders(config.apiKey), }, body: JSON.stringify({ model: modelId, input: text, voice, speed: config.speed || 1.0, response_format: config.format || 'wav', }), signal, }); if (!response.ok) { throwIfTtsRateLimited('Lemonade', response.status, response.headers?.get('retry-after')); throw new Error(`Lemonade TTS API error: ${await readTTSApiError(response)}`); } return await validateTTSAudioResponse(response, 'Lemonade', config.format || 'wav'); } /** * VoxCPM2 TTS implementation. * * OpenMAIC keeps one internal VoxCPM request shape, then adapts it to the * selected official backend protocol. */ async function generateVoxCPMTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = (config.baseUrl || TTS_PROVIDERS['voxcpm-tts'].defaultBaseUrl || '').replace( /\/$/, '', ); if (!baseUrl) { throw new Error('VoxCPM base URL is required'); } const options = (config.providerOptions || {}) as VoxCPMProviderOptions; const backend = normalizeVoxCPMBackend(options.backend); const voicePrompt = options.voicePrompt || (config.voice && config.voice !== 'default' && config.voice !== VOXCPM_AUTO_VOICE_ID ? config.voice : undefined); // A registered voice carries timbre by id, so no voice prompt is required. const registeredVoiceId = options.registeredVoiceId?.trim() || undefined; if (config.voice === VOXCPM_AUTO_VOICE_ID && !voicePrompt && !registeredVoiceId) { throw new Error('VoxCPM Auto Voice requires agent context'); } const cfgValue = options.cfgValue ?? 2.0; const inferenceTimesteps = options.inferenceTimesteps ?? 10; const normalize = options.normalize ?? false; const denoise = options.denoise ?? false; const usePromptContinuation = Boolean(options.promptText?.trim() && options.referenceAudioBase64); const request = { targetText: usePromptContinuation ? text : buildVoxCPMTargetText(text, voicePrompt), rawText: text, registeredVoiceId, voicePrompt, promptText: options.promptText, cfgValue, inferenceTimesteps, normalize, denoise, referenceAudioBase64: options.referenceAudioBase64, referenceAudioMimeType: options.referenceAudioMimeType, referenceAudioName: options.referenceAudioName, }; const response = backend === 'nano-vllm' ? await postVoxCPMNanoVLLM(baseUrl, request, config.apiKey, signal, config) : backend === 'python-api' ? await postVoxCPMPythonAPI(baseUrl, request, config.apiKey, signal, config) : await postVoxCPMVLLMOmni(baseUrl, request, config, signal); if (!response.ok) { throwIfTtsRateLimited('VoxCPM', response.status, response.headers?.get('retry-after')); throw new Error(`VoxCPM TTS API error: ${await readTTSApiError(response)}`); } return await validateTTSAudioResponse(response, 'VoxCPM', 'wav'); } function buildVoxCPMTargetText(text: string, voicePrompt?: string): string { const prompt = voicePrompt ?.replace(/[\p{C}]+/gu, ' ') .replace(/[()()]/gu, '') .replace(/\s+/gu, ' ') .trim(); return prompt ? `(${prompt})${text}` : text; } function findFirstNonWhitespaceByte(bytes: Uint8Array): number | null { let i = 0; // Skip UTF-8 BOM if present: 0xEF, 0xBB, 0xBF if (bytes.length >= 3 && bytes[0] === 0xef && bytes[1] === 0xbb && bytes[2] === 0xbf) { i = 3; } for (; i < bytes.length; i++) { const b = bytes[i]; // Skip ASCII whitespace: space (0x20), tab (0x09), newline (0x0A), carriage return (0x0D) if (b !== 0x20 && b !== 0x09 && b !== 0x0a && b !== 0x0d) { return b; } } return null; } /** * Shared validator for TTS audio responses. * * Rejects 200 responses containing non-audio bodies (HTML pages, JSON responses, * text/plain, or empty/blank responses) with a typed TTSInvalidResponseError (502) before * bytes can be treated as narration, billed, or saved. * * Headerless audio formats like raw PCM, μ-law, and A-law have no magic numbers and ~1.2% * of valid audio chunks start with '<', '{', or '['. Therefore, when the response * Content-Type indicates audio/*, the leading-byte sniff is skipped entirely (#1395). */ async function validateTTSAudioResponse( response: Response, provider: string, fallbackFormat = 'mp3', ): Promise { const contentType = response.headers.get('content-type') || ''; const lowerContentType = contentType.toLowerCase(); const arrayBuffer = await response.arrayBuffer(); const bytes = new Uint8Array(arrayBuffer); if (bytes.byteLength === 0) { throw new TTSInvalidResponseError( provider, `${provider} TTS returned an empty audio response (0 bytes)`, ); } // When Content-Type begins with audio/*, trust it as audio without inspecting // leading bytes, avoiding false positives on headerless formats (PCM, μ-law, A-law) // whose raw samples can start with '<', '{', or '[' (#1395). if (lowerContentType.startsWith('audio/')) { return { audio: bytes, format: getAudioResponseFormat(contentType, fallbackFormat), }; } const firstByte = findFirstNonWhitespaceByte(bytes); if (firstByte === null) { throw new TTSInvalidResponseError( provider, `${provider} TTS returned a blank/whitespace-only response`, ); } const isHtml = lowerContentType.includes('text/html') || lowerContentType.includes('application/xhtml+xml') || firstByte === 0x3c; // '<' if (isHtml) { const textSnippet = new TextDecoder('utf-8').decode(bytes).slice(0, 300); log.warn(`${provider} TTS returned HTML instead of audio: ${textSnippet}`); throw new TTSInvalidResponseError( provider, `${provider} TTS returned an HTML response instead of audio. Check provider base URL.`, ); } const isJson = lowerContentType.includes('application/json') || firstByte === 0x7b || // '{' firstByte === 0x5b; // '[' if (isJson) { const textSnippet = new TextDecoder('utf-8').decode(bytes).slice(0, 300); log.warn(`${provider} TTS returned JSON instead of audio: ${textSnippet}`); throw new TTSInvalidResponseError( provider, `${provider} TTS returned a JSON response instead of audio.`, ); } if (lowerContentType.includes('text/plain')) { const textSnippet = new TextDecoder('utf-8').decode(bytes).slice(0, 300); log.warn(`${provider} TTS returned text/plain instead of audio: ${textSnippet}`); throw new TTSInvalidResponseError( provider, `${provider} TTS returned text/plain instead of audio.`, ); } return { audio: bytes, format: getAudioResponseFormat(contentType, fallbackFormat), }; } function getAudioResponseFormat(contentType: string, fallbackFormat = 'mp3'): string { const lower = contentType.toLowerCase(); if (lower.includes('audio/wav') || lower.includes('audio/x-wav')) return 'wav'; if (lower.includes('audio/mpeg') || lower.includes('audio/mp3')) return 'mp3'; if (lower.includes('audio/flac')) return 'flac'; if (lower.includes('audio/ogg')) return 'ogg'; if (lower.includes('audio/webm')) return 'webm'; if (lower.includes('audio/aac')) return 'aac'; if (lower.includes('audio/opus')) return 'opus'; return fallbackFormat; } function getVoxCPMAudioFormat(mimeType?: string, fileName?: string): string { const lowerName = fileName?.toLowerCase() || ''; if (mimeType?.includes('wav') || lowerName.endsWith('.wav')) return 'wav'; if (mimeType?.includes('mpeg') || mimeType?.includes('mp3') || lowerName.endsWith('.mp3')) { return 'mp3'; } if (mimeType?.includes('flac') || lowerName.endsWith('.flac')) return 'flac'; if (mimeType?.includes('ogg') || lowerName.endsWith('.ogg')) return 'ogg'; if (mimeType?.includes('webm') || lowerName.endsWith('.webm')) return 'webm'; return 'wav'; } function getVLLMOmniSpeechUrl(baseUrl: string): string { return baseUrl.endsWith('/v1') ? `${baseUrl}/audio/speech` : `${baseUrl}/v1/audio/speech`; } function getVLLMOmniModelId(config: TTSModelConfig): string { const modelId = config.modelId; // VoxCPM2 is the operator-declared id; vLLM-Omni requires this fixed internal // alias. The rewrite is deterministic and cannot be selected by a request. if (!modelId || modelId === 'VoxCPM2') return VOXCPM_VLLM_MODEL_ID; return modelId; } function getBackendAuthHeaders(apiKey?: string): Record { return apiKey?.trim() ? { Authorization: `Bearer ${apiKey.trim()}` } : {}; } async function postVoxCPMVLLMOmni( baseUrl: string, params: { targetText: string; rawText?: string; registeredVoiceId?: string; promptText?: string; referenceAudioBase64?: string; referenceAudioMimeType?: string; referenceAudioName?: string; }, config: TTSModelConfig, signal: AbortSignal, ): Promise { const payload: Record = { model: getVLLMOmniModelId(config), input: params.targetText, voice: 'default', response_format: 'wav', stream: false, }; if (params.registeredVoiceId) { // A registered voice carries timbre by id (pre-encoded latents): reference it // directly and send the raw text — no inline voice-design prompt or ref_audio. payload.voice = params.registeredVoiceId; payload.input = params.rawText ?? params.targetText; } else if (params.referenceAudioBase64) { const referenceAudio = getVoxCPMDataAudioUrl( params.referenceAudioBase64, params.referenceAudioMimeType, params.referenceAudioName, ); payload.ref_audio = referenceAudio; if (params.promptText?.trim()) { payload.prompt_audio = referenceAudio; payload.prompt_text = params.promptText.trim(); } } return ttsFetch(config, getVLLMOmniSpeechUrl(baseUrl), { method: 'POST', headers: { 'Content-Type': 'application/json; charset=utf-8', ...getBackendAuthHeaders(config.apiKey), }, body: JSON.stringify(payload), signal, }); } function getVoxCPMDataAudioUrl(base64: string, mimeType?: string, fileName?: string): string { const format = getVoxCPMAudioFormat(mimeType, fileName); const mediaType = mimeType?.trim() || (format === 'mp3' ? 'audio/mpeg' : format === 'flac' ? 'audio/flac' : format === 'ogg' ? 'audio/ogg' : format === 'webm' ? 'audio/webm' : 'audio/wav'); return `data:${mediaType};base64,${base64}`; } function base64ToBlob(base64: string, mimeType?: string): Blob { const binary = atob(base64); const bytes = new Uint8Array(binary.length); for (let index = 0; index < binary.length; index++) { bytes[index] = binary.charCodeAt(index); } return new Blob([bytes], { type: mimeType || 'audio/wav' }); } async function postVoxCPMPythonAPI( baseUrl: string, params: { targetText: string; promptText?: string; cfgValue: number; inferenceTimesteps: number; normalize: boolean; denoise: boolean; referenceAudioBase64?: string; referenceAudioMimeType?: string; referenceAudioName?: string; }, apiKey?: string, signal?: AbortSignal, target?: AudioEndpointTarget, ): Promise { const formData = new FormData(); formData.set('text', params.targetText); formData.set('cfg_value', String(params.cfgValue)); formData.set('inference_timesteps', String(params.inferenceTimesteps)); formData.set('normalize', String(params.normalize)); formData.set('denoise', String(params.denoise)); if (params.referenceAudioBase64) { const audioBlob = base64ToBlob(params.referenceAudioBase64, params.referenceAudioMimeType); const audioName = params.referenceAudioName || 'reference.wav'; formData.set('reference_audio', audioBlob, audioName); if (params.promptText?.trim()) { formData.set('prompt_audio', audioBlob, audioName); formData.set('prompt_text', params.promptText.trim()); } } return ttsFetch(target, `${baseUrl}/tts/upload`, { method: 'POST', headers: getBackendAuthHeaders(apiKey), body: formData, signal, }); } async function postVoxCPMNanoVLLM( baseUrl: string, params: { targetText: string; promptText?: string; cfgValue: number; referenceAudioBase64?: string; referenceAudioMimeType?: string; referenceAudioName?: string; }, apiKey?: string, signal?: AbortSignal, target?: AudioEndpointTarget, ): Promise { const payload: Record = { target_text: params.targetText, cfg_value: params.cfgValue, }; if (params.referenceAudioBase64) { const format = getVoxCPMAudioFormat(params.referenceAudioMimeType, params.referenceAudioName); payload.ref_audio_wav_base64 = params.referenceAudioBase64; payload.ref_audio_wav_format = format; if (params.promptText?.trim()) { payload.prompt_wav_base64 = params.referenceAudioBase64; payload.prompt_wav_format = format; payload.prompt_text = params.promptText.trim(); } } return ttsFetch(target, `${baseUrl}/generate`, { method: 'POST', headers: { 'Content-Type': 'application/json; charset=utf-8', ...getBackendAuthHeaders(apiKey), }, body: JSON.stringify(payload), signal, }); } async function readTTSApiError(response: Response): Promise { const text = await response.text().catch(() => response.statusText); if (!text) return response.statusText; try { const json = JSON.parse(text) as { detail?: unknown; error?: { message?: string } | string; }; if (typeof json.detail === 'string') return json.detail; if (typeof json.error === 'string') return json.error; if (json.error?.message) return json.error.message; } catch { // Fall through to raw text. } return text; } /** * Azure TTS implementation (direct API call with SSML) */ async function generateAzureTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['azure-tts'].defaultBaseUrl; const voiceLocale = resolveAzureVoiceLocale(config.voice); // Build SSML const rate = config.speed ? `${((config.speed - 1) * 100).toFixed(0)}%` : '0%'; const ssml = ` ${escapeXml(text)} `.trim(); const response = await ttsFetch(config, `${baseUrl}/cognitiveservices/v1`, { method: 'POST', headers: { 'Ocp-Apim-Subscription-Key': config.apiKey!, 'Content-Type': 'application/ssml+xml; charset=utf-8', 'X-Microsoft-OutputFormat': 'audio-16khz-128kbitrate-mono-mp3', }, body: ssml, signal, }); if (!response.ok) { throwIfTtsRateLimited('Azure', response.status, response.headers?.get('retry-after')); throw new Error(`Azure TTS API error: ${response.statusText}`); } return await validateTTSAudioResponse(response, 'Azure', 'mp3'); } /** Resolve the BCP-47 locale encoded by an Azure voice identifier. */ function resolveAzureVoiceLocale(voice: string): string { const configuredVoice = TTS_PROVIDERS['azure-tts'].voices.find(({ id }) => id === voice); if (configuredVoice?.language) return configuredVoice.language; // Azure voice IDs conventionally start with a BCP-47 locale (for example, // `en-US-JennyNeural` or `sr-Latn-RS-SophieNeural`). Preserve optional // script and variant subtags for voices outside the small configured list // while retaining the existing Chinese default for an unrecognised ID. return ( voice.match( /^[a-z]{2,3}(?:-[A-Z][a-z]{3})?-(?:[A-Z]{2}|\d{3})(?:-(?:[a-z0-9]{5,8}|\d[a-z0-9]{3}))*(?=-[A-Z]|$)/, )?.[0] ?? 'zh-CN' ); } /** * GLM TTS implementation (GLM API) */ async function generateGLMTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['glm-tts'].defaultBaseUrl; const response = await ttsFetch(config, `${baseUrl}/audio/speech`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: config.modelId || 'glm-tts', input: text, voice: config.voice, speed: config.speed || 1.0, volume: 1.0, response_format: 'wav', }), signal, }); if (!response.ok) { throwIfTtsRateLimited('GLM', response.status, response.headers?.get('retry-after')); const errorText = await response.text().catch(() => response.statusText); let errorMessage = `GLM TTS API error: ${errorText}`; try { const errorJson = JSON.parse(errorText); if (errorJson.error?.message) { errorMessage = `GLM TTS API error: ${errorJson.error.message} (code: ${errorJson.error.code})`; } } catch { // If not JSON, use the text as is } throw new Error(errorMessage); } return await validateTTSAudioResponse(response, 'GLM', 'wav'); } /** * Qwen TTS implementation (DashScope API - Qwen3 TTS Flash) */ async function generateQwenTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['qwen-tts'].defaultBaseUrl; const cloneVoice = isQwenCloneVoice(config.voice, config.voices); if (cloneVoice) { try { return await synthesizeQwenVoiceClone( { apiKey: config.apiKey, baseUrl, // Server callers resolve and authorize the actual target before dispatch. targetModel: config.modelId, publicOnly: config.publicOnly, managed: config.managed, }, text, config.voice, config.speed, signal, ); } catch (error) { if (error instanceof QwenVoiceCloneError && error.code === 'QWEN_VC_VOICE_NOT_FOUND') { evictQwenVoiceRegistrationMemo(config.voice); } throw error; } } // Calculate speed: Qwen3 uses rate parameter from -500 to 500 // speed 1.0 = rate 0, speed 2.0 = rate 500, speed 0.5 = rate -250 const rate = Math.round(((config.speed || 1.0) - 1.0) * 500); const modelId = resolveTTSModelForVoice('qwen-tts', config.voice, config.modelId, config.voices); const response = await ttsFetch( config, `${baseUrl}/services/aigc/multimodal-generation/generation`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ model: modelId || 'qwen3-tts-flash', input: { text, voice: config.voice, language_type: 'Chinese', // Default to Chinese, can be made configurable }, parameters: { rate, // Speech rate from -500 to 500 }, }), signal, }, ); if (!response.ok) { throwIfTtsRateLimited('Qwen', response.status, response.headers?.get('retry-after')); const errorText = await response.text().catch(() => response.statusText); throw new QwenTTSError(`Qwen TTS request failed: ${errorText}`, response.status); } const data = await response.json(); // Check for audio URL in response if (!data.output?.audio?.url) { throw new QwenTTSError('Qwen TTS returned no audio URL.'); } // Download audio from URL let downloaded; try { downloaded = await downloadAudio(data.output.audio.url, signal, baseUrl, config.publicOnly); } catch (error) { if (error instanceof QwenVoiceCloneError) { const host = (() => { try { return new URL(String(data.output.audio.url)).hostname || 'unknown'; } catch { return 'invalid'; } })(); throw new QwenTTSError( error.code === 'QWEN_VC_AUDIO_URL_INVALID' ? `The generated Qwen audio URL host "${host}" is not allowed.` : 'The generated Qwen audio could not be downloaded.', error.httpStatus, ); } throw error; } return { audio: downloaded.bytes, format: 'wav', // Qwen3 TTS returns WAV format }; } /** * MiniMax rate / concurrency business codes. These arrive as HTTP 200 with * `base_resp.status_code` set, including when `data.audio` is also present. * 1002 RPM, 1039 token limit, 1041 connection limit, 2045 rate-growth limit. */ const MINIMAX_RATE_LIMIT_STATUS_CODES = new Set([1002, 1039, 1041, 2045]); /** * MiniMax plan and balance exhaustion, as opposed to its rate limits: * 1008 insufficient balance, 2056 usage limit exceeded (the plan's window). * https://platform.minimax.io/docs/api-reference/errorcode */ const MINIMAX_QUOTA_STATUS_CODES = new Set([1008, 2056]); function minimaxStatusCode(value: unknown): number | undefined { if (typeof value === 'number' && Number.isFinite(value)) return value; if (typeof value === 'string' && value.trim() !== '') { const parsed = Number(value); if (Number.isFinite(parsed)) return parsed; } return undefined; } /** The `base_resp.status_code` of a MiniMax error response body, if it has one. */ function minimaxBodyStatusCode(body: string): number | undefined { try { const parsed = JSON.parse(body) as { base_resp?: { status_code?: unknown } } | null; return minimaxStatusCode(parsed?.base_resp?.status_code); } catch { return undefined; } } /** A MiniMax quota code as the provider-neutral quota error; undefined for any other code. */ function minimaxQuotaError( statusCode: number | undefined, message: string, ): ProviderQuotaExhaustedError | undefined { return statusCode !== undefined && MINIMAX_QUOTA_STATUS_CODES.has(statusCode) ? new ProviderQuotaExhaustedError('MiniMax', message) : undefined; } /** Any non-zero MiniMax `base_resp.status_code` is a failure, even with audio bytes. */ function throwIfMiniMaxBaseRespFailed( data: { base_resp?: { status_code?: unknown; status_msg?: unknown } } | null | undefined, retryAfterHeader?: string | null, ): void { const statusCode = minimaxStatusCode(data?.base_resp?.status_code); if (statusCode === undefined || statusCode === 0) return; const statusMsg = typeof data?.base_resp?.status_msg === 'string' && data.base_resp.status_msg.trim() ? data.base_resp.status_msg.trim() : `status ${statusCode}`; if (MINIMAX_RATE_LIMIT_STATUS_CODES.has(statusCode)) { throw new TTSRateLimitError( 'MiniMax', `MiniMax TTS rate limit exceeded: ${statusMsg}`, parseRetryAfterMs(retryAfterHeader), ); } const message = `MiniMax TTS API error (${statusCode}): ${statusMsg}`; throw minimaxQuotaError(statusCode, message) ?? new Error(message); } /** * MiniMax TTS implementation (synchronous HTTP API) */ async function generateMiniMaxTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = (config.baseUrl || TTS_PROVIDERS['minimax-tts'].defaultBaseUrl || '').replace( /\/$/, '', ); const response = await ttsFetch(config, `${baseUrl}/v1/t2a_v2`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', ...appAttributionHeaders(baseUrl), }, body: JSON.stringify({ model: config.modelId || 'speech-2.8-hd', text, stream: false, output_format: 'hex', voice_setting: { voice_id: config.voice, speed: config.speed || 1.0, vol: 1, pitch: 0, }, audio_setting: { sample_rate: 32000, bitrate: 128000, format: config.format || 'mp3', channel: 1, }, language_boost: 'auto', }), signal, }); if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); const message = `MiniMax TTS API error: ${errorText}`; // A quota code in the body says what the HTTP status does not; an HTTP 429 // keeps its rate-limit type and carries the quota error as its cause. const quota = minimaxQuotaError(minimaxBodyStatusCode(errorText), message); throwIfTtsRateLimited( 'MiniMax', response.status, response.headers?.get('retry-after'), quota ? { cause: quota } : undefined, ); throw quota ?? new Error(message); } const data = await response.json(); throwIfMiniMaxBaseRespFailed(data, response.headers?.get('retry-after')); const hexAudio = data?.data?.audio; if (!hexAudio || typeof hexAudio !== 'string') { throw new Error(`MiniMax TTS error: No audio returned. Response: ${JSON.stringify(data)}`); } const cleanedHex = hexAudio.trim(); if (cleanedHex.length % 2 !== 0) { throw new Error('MiniMax TTS error: invalid hex audio payload length'); } const audio = new Uint8Array( cleanedHex.match(/.{1,2}/g)?.map((byte: string) => parseInt(byte, 16)) || [], ); return { audio, format: data?.extra_info?.audio_format || config.format || 'mp3', }; } /** * Google Gemini TTS via the Interactions API. * Gemini 3.8 returns WAV; earlier preview models return raw mono s16le PCM. */ async function generateGoogleTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = (config.baseUrl || TTS_PROVIDERS['google-tts'].defaultBaseUrl || '').replace( /\/$/, '', ); const modelId = config.modelId || TTS_PROVIDERS['google-tts'].defaultModelId; const voice = config.voice || DEFAULT_TTS_VOICES['google-tts'] || 'Kore'; const response = await ttsFetch(config, `${baseUrl}/interactions`, { method: 'POST', headers: { 'x-goog-api-key': config.apiKey!, 'Content-Type': 'application/json; charset=utf-8', ...appAttributionHeaders(baseUrl), }, body: JSON.stringify({ model: modelId, input: text, response_format: { type: 'audio' }, generation_config: { speech_config: [{ voice }], }, }), signal, }); if (!response.ok) { throwIfTtsRateLimited('Google', response.status, response.headers?.get('retry-after')); const errorText = await response.text().catch(() => response.statusText); throw new Error(`Google Gemini TTS API error: ${errorText || response.statusText}`); } const payload = (await response.json().catch(() => null)) as Record | null; const audio = extractGoogleInteractionsAudio(payload); if (!audio) { throw new Error('Google Gemini TTS API error: response missing audio data'); } const bytes = new Uint8Array(Buffer.from(audio.data, 'base64')); if (bytes.byteLength === 0) { throw new Error('Google Gemini TTS API error: empty audio payload'); } const mimeType = audio.mimeType?.split(';')[0].trim().toLowerCase(); if (mimeType === 'audio/wav') { return { audio: bytes, format: 'wav' }; } // Preserve the legacy PCM fallback for responses without a MIME type. if (mimeType && mimeType !== 'audio/l16') { throw new Error('Google Gemini TTS API error: unsupported audio format'); } return { audio: pcmS16leMonoToWav(bytes, sampleRateFromL16Mime(audio.mimeType)), format: 'wav', }; } /** * Interactions REST responses put audio on `steps[].content[]`, not on the * SDK convenience field `output_audio`. See speech-generation single-speaker docs. */ function extractGoogleInteractionsAudio( payload: Record | null, ): { data: string; mimeType?: string } | null { if (!payload || !Array.isArray(payload.steps)) return null; for (const step of payload.steps) { if (!step || typeof step !== 'object') continue; const content = (step as { content?: unknown }).content; if (!Array.isArray(content)) continue; for (const part of content) { if (!part || typeof part !== 'object') continue; const block = part as { type?: unknown; data?: unknown; mime_type?: unknown }; if (block.type !== 'audio' || typeof block.data !== 'string' || block.data.length === 0) { continue; } return { data: block.data, mimeType: typeof block.mime_type === 'string' ? block.mime_type : undefined, }; } } return null; } /** `audio/l16;rate=24000` → 24000. Missing or unparseable rate stays at 24 kHz. */ function sampleRateFromL16Mime(mime: unknown): number { if (typeof mime !== 'string') return 24_000; const match = /(?:^|[;\s])rate=(\d+)/i.exec(mime); const rate = match ? Number(match[1]) : NaN; return Number.isFinite(rate) && rate > 0 ? rate : 24_000; } /** * ElevenLabs TTS implementation (direct API call with voice-specific endpoint) */ async function generateElevenLabsTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = config.baseUrl || TTS_PROVIDERS['elevenlabs-tts'].defaultBaseUrl; const requestedFormat = config.format || 'mp3'; const clampedSpeed = Math.min(1.2, Math.max(0.7, config.speed || 1.0)); const outputFormatMap: Record = { mp3: 'mp3_44100_128', opus: 'opus_48000_96', pcm: 'pcm_44100', wav: 'wav_44100', ulaw: 'ulaw_8000', alaw: 'alaw_8000', }; const outputFormat = outputFormatMap[requestedFormat] || outputFormatMap.mp3; const response = await ttsFetch( config, `${baseUrl}/text-to-speech/${encodeURIComponent(config.voice)}?output_format=${outputFormat}`, { method: 'POST', headers: { 'xi-api-key': config.apiKey!, 'Content-Type': 'application/json; charset=utf-8', }, body: JSON.stringify({ text, model_id: config.modelId || 'eleven_multilingual_v2', voice_settings: { stability: 0.5, similarity_boost: 0.75, speed: clampedSpeed, }, }), signal, }, ); if (!response.ok) { throwIfTtsRateLimited('ElevenLabs', response.status, response.headers?.get('retry-after')); const errorText = await response.text().catch(() => response.statusText); throw new Error(`ElevenLabs TTS API error: ${errorText || response.statusText}`); } return await validateTTSAudioResponse(response, 'ElevenLabs', requestedFormat); } // Re-export from constants for convenience export { getAllTTSProviders, getTTSProvider, getTTSVoices } from './constants'; /** * Doubao TTS 2.0 implementation (Volcengine Seed-TTS 2.0). * * Two auth modes, distinguished by the API key shape — Volcengine exposes * Seed-TTS as two separate products that do NOT share credentials or endpoints * (verified: a plan key 401s on the normal endpoint, and the plan endpoint * rejects Bearer auth): * - Standalone speech console: `appId:accessKey` → normal endpoint * (.../api/v3/tts/unidirectional) with `X-Api-App-Id` + `X-Api-Access-Key`. * - Ark Agent Plan: a single `ark-...` plan key → plan endpoint * (.../api/plan/tts/unidirectional, carried in config.baseUrl) with * `X-Api-Key`. Lit up via the Token Plan one-click setup. * The endpoint and auth header are bound together, so we pick both from the key * shape — never a normal endpoint with X-Api-Key, or vice versa. */ async function generateDoubaoTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const rawKey = config.apiKey || ''; if (!rawKey) { throw new Error( 'Doubao TTS requires an API key: an Agent Plan key, or "appId:accessKey" from the Volcengine speech console.', ); } const colonIdx = rawKey.indexOf(':'); // A colon means the classic appId:accessKey pair; otherwise treat the whole // value as an Agent Plan single key (X-Api-Key auth on the /plan endpoint). const isPlanKey = colonIdx < 0; const appId = isPlanKey ? '' : rawKey.slice(0, colonIdx); const accessKey = isPlanKey ? '' : rawKey.slice(colonIdx + 1); // A colon with an empty half is a malformed pair — fail clearly rather than // sending an empty appId/accessKey header that the API rejects opaquely. if (!isPlanKey && (!appId || !accessKey)) { throw new Error( 'Doubao TTS appId:accessKey is malformed — both halves are required (or use an Agent Plan key).', ); } const baseUrl = config.baseUrl || TTS_PROVIDERS['doubao-tts'].defaultBaseUrl; const speechRate = Math.round(((config.speed || 1.0) - 1.0) * 100); const authHeaders: Record = isPlanKey ? { 'X-Api-Key': rawKey } : { 'X-Api-App-Id': appId, 'X-Api-Access-Key': accessKey }; const response = await ttsFetch(config, `${baseUrl}/unidirectional`, { method: 'POST', headers: { 'Content-Type': 'application/json', ...authHeaders, 'X-Api-Resource-Id': 'seed-tts-2.0', }, body: JSON.stringify({ user: { uid: 'openmaic' }, req_params: { text, speaker: config.voice, audio_params: { format: 'mp3', sample_rate: 24000, speech_rate: speechRate, }, }, }), signal, }); if (!response.ok) { throwIfTtsRateLimited('Doubao', response.status, response.headers?.get('retry-after')); const errorText = await response.text().catch(() => response.statusText); throw new Error(`Doubao TTS API error (${response.status}): ${errorText}`); } const responseText = await response.text(); const audioChunks: Uint8Array[] = []; // Doubao streams a run of concatenated JSON objects with no delimiter. Split // them string-aware (see splitConcatenatedJsonObjects) — a naive `{`/`}` depth // counter miscounts braces that appear inside a string value (e.g. an error // `message` containing `}`), which corrupts the object boundaries. for (const objectText of splitConcatenatedJsonObjects(responseText)) { let chunk: { code: number; message?: string; data?: string }; try { chunk = JSON.parse(objectText); } catch { continue; } if (chunk.code === 0 && chunk.data) { audioChunks.push(new Uint8Array(Buffer.from(chunk.data, 'base64'))); } else if (chunk.code === 20000000) { break; } else if (chunk.code && chunk.code !== 0) { if (chunk.code === 45000000 || chunk.code === 45000292) { throw new TTSRateLimitError('doubao-tts', chunk.message || 'concurrency quota exceeded'); } throw new Error(`Doubao TTS error: ${chunk.message || 'unknown'} (code: ${chunk.code})`); } } if (audioChunks.length === 0) { throw new Error('Doubao TTS: no audio data received'); } const totalLength = audioChunks.reduce((sum, c) => sum + c.length, 0); const combined = new Uint8Array(totalLength); let offset = 0; for (const chunk of audioChunks) { combined.set(chunk, offset); offset += chunk.length; } return { audio: combined, format: 'mp3' }; } /** * Escape XML special characters for SSML */ function escapeXml(text: string): string { return text .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); } /** * Map a numeric speed to a graded natural-language pacing instruction. * MiMo TTS takes speed only as a style instruction, not a request parameter, * so the slider range maps to graded wording: 1.05 and 2.0 must not produce * the same request. */ function xiaomiPacingInstruction(speed: number): string | null { if (speed > 1.0) { const pct = Math.round((speed - 1) * 100); if (pct >= 60) return '说得非常快,语速很快。'; if (pct >= 25) return '说得快一些,语速偏快。'; return '语速稍微快一点。'; } if (speed < 1.0) { const pct = Math.round((1 - speed) * 100); if (pct >= 40) return '说得非常慢,语速很慢。'; if (pct >= 15) return '说得慢一些,语速偏慢。'; return '语速稍微慢一点。'; } return null; } /** * Xiaomi MiMo TTS implementation. * * MiMo TTS does NOT expose an OpenAI-style /audio/speech endpoint. It reuses * the chat/completions protocol: the text to synthesize goes in an `assistant` * message (a hard provider rule), optional style instructions go in a `user` * message, and the audio payload comes back base64-encoded at * `choices[0].message.audio.data`. Numeric speed is not a request parameter, * so a non-default speed is translated into a graded pacing instruction * (`xiaomiPacingInstruction`). */ async function generateXiaomiTTS( config: TTSModelConfig, text: string, signal: AbortSignal, ): Promise { const baseUrl = (config.baseUrl || TTS_PROVIDERS['xiaomi-tts'].defaultBaseUrl || '').replace( /\/+$/, '', ); const messages: Array<{ role: string; content: string }> = []; const pacing = xiaomiPacingInstruction(config.speed ?? 1.0); if (pacing) { messages.push({ role: 'user', content: pacing }); } messages.push({ role: 'assistant', content: text }); const response = await ttsFetch(config, `${baseUrl}/chat/completions`, { method: 'POST', headers: { Authorization: `Bearer ${config.apiKey}`, 'Content-Type': 'application/json; charset=utf-8', ...appAttributionHeaders(baseUrl), }, body: JSON.stringify({ model: config.modelId || TTS_PROVIDERS['xiaomi-tts'].defaultModelId, messages, audio: { format: 'wav', voice: config.voice || 'mimo_default', }, }), signal, }); if (!response.ok) { throwIfTtsRateLimited('Xiaomi MiMo', response.status, response.headers.get('retry-after')); const error = await response.json().catch(() => ({ error: { message: response.statusText } })); throw new Error(`Xiaomi MiMo TTS API error: ${error.error?.message || response.statusText}`); } const data = await response.json(); const audioData = data?.choices?.[0]?.message?.audio?.data; if (typeof audioData !== 'string' || !audioData) { throw new TTSInvalidResponseError('Xiaomi MiMo', 'Xiaomi MiMo TTS API returned no audio data'); } // Node's base64 decoder is lenient: whitespace or malformed input silently // decodes to zero or arbitrary bytes. A "success" carrying those would be // billed and persisted as an unplayable asset, so validate the payload. const audio = Buffer.from(audioData, 'base64'); if ( audio.byteLength < 44 || !audio.subarray(0, 4).equals(Buffer.from('RIFF')) || !audio.subarray(8, 12).equals(Buffer.from('WAVE')) ) { throw new TTSInvalidResponseError( 'Xiaomi MiMo', `Xiaomi MiMo TTS returned a ${audio.byteLength}-byte payload that is not a valid WAV file`, ); } return { audio: new Uint8Array(audio), format: 'wav', }; }