/** * Media Generation Orchestrator * * Dispatches media generation API calls for all mediaGenerations across outlines. * Runs entirely on the frontend — calls /api/generate/image and /api/generate/video, * fetches result blobs, stores in IndexedDB, and updates the Zustand store. */ import { useMediaGenerationStore } from '@/lib/store/media-generation'; import { useSettingsStore } from '@/lib/store/settings'; import { db, mediaFileKey } from '@/lib/utils/database'; import type { SceneOutline } from '@/lib/types/generation'; import type { MediaGenerationRequest } from '@/lib/media/types'; import { fetchProxiedMediaUrl } from '@/lib/media/proxy-media-cache'; import { createLogger } from '@/lib/logger'; const log = createLogger('MediaOrchestrator'); /** Error with a structured errorCode from the API */ class MediaApiError extends Error { errorCode?: string; constructor(message: string, errorCode?: string) { super(message); this.errorCode = errorCode; } } function createAbortError(): Error { if (typeof DOMException !== 'undefined') return new DOMException('Aborted', 'AbortError'); return Object.assign(new Error('Aborted'), { name: 'AbortError' }); } function throwIfAborted(signal?: AbortSignal): void { if (signal?.aborted) throw createAbortError(); } /** * Launch media generation for all mediaGenerations declared in outlines. * Runs in parallel with content/action generation — does not block. */ export async function generateMediaForOutlines( outlines: SceneOutline[], stageId: string, abortSignal?: AbortSignal, ): Promise { const settings = useSettingsStore.getState(); const store = useMediaGenerationStore.getState(); // Collect all media requests const allRequests: MediaGenerationRequest[] = []; for (const outline of outlines) { if (!outline.mediaGenerations) continue; for (const mg of outline.mediaGenerations) { // Filter by enabled flags if (mg.type === 'image' && !settings.imageGenerationEnabled) continue; if (mg.type === 'video' && !settings.videoGenerationEnabled) continue; // Skip already completed or permanently failed (restored from DB) const existing = store.getTask(mg.elementId); if (existing?.status === 'done' || existing?.status === 'failed') continue; allRequests.push(mg); } } if (allRequests.length === 0) return; // Enqueue all as pending useMediaGenerationStore.getState().enqueueTasks(stageId, allRequests); // Process requests serially — image/video APIs have limited concurrency for (const req of allRequests) { if (abortSignal?.aborted) break; await generateSingleMedia(req, stageId, abortSignal); } } /** * Retry a single failed media task. */ export async function retryMediaTask( elementId: string, _target?: { readonly elementId: string; readonly sceneId?: string; readonly slideId?: string }, ): Promise { const store = useMediaGenerationStore.getState(); const task = store.getTask(elementId); if (!task || task.status !== 'failed') return; // Check if the corresponding generation type is still enabled in global settings const settings = useSettingsStore.getState(); if (task.type === 'image' && !settings.imageGenerationEnabled) { store.markFailed(elementId, 'Generation disabled', 'GENERATION_DISABLED'); return; } if (task.type === 'video' && !settings.videoGenerationEnabled) { store.markFailed(elementId, 'Generation disabled', 'GENERATION_DISABLED'); return; } // Remove persisted failure record from DB so a fresh result can be written const dbKey = mediaFileKey(task.stageId, elementId); await db.mediaFiles.delete(dbKey).catch(() => {}); store.markPendingForRetry(elementId); await generateSingleMedia( { type: task.type, prompt: task.prompt, elementId: task.elementId, aspectRatio: task.params.aspectRatio as MediaGenerationRequest['aspectRatio'], style: task.params.style, }, task.stageId, ); } /** Build the renderer retry scope while classic retries remain placeholder-keyed. */ export function mediaRetryTarget( elementId: string, sceneId: string | undefined, sceneData: unknown, ): { elementId: string; sceneId?: string; slideId?: string } { const slideId = sceneData && typeof sceneData === 'object' && 'canvas' in sceneData ? (sceneData as { canvas?: { id?: string } }).canvas?.id : undefined; return { elementId, ...(sceneId ? { sceneId } : {}), ...(slideId ? { slideId } : {}) }; } // ==================== Internal ==================== async function generateSingleMedia( req: MediaGenerationRequest, stageId: string, abortSignal?: AbortSignal, ): Promise { const store = useMediaGenerationStore.getState(); store.markGenerating(req.elementId); try { const paramsJson = JSON.stringify({ aspectRatio: req.aspectRatio, style: req.style, }); if (req.type === 'image') { const result = await callImageApi(req, stageId, abortSignal); // CDN path: server already uploaded to OSS if (result.ossUrl) { throwIfAborted(abortSignal); await db.mediaFiles.put({ id: mediaFileKey(stageId, req.elementId), stageId, type: 'image', blob: new Blob([]), mimeType: 'image/png', size: 0, ossKey: result.ossUrl, prompt: req.prompt, params: paramsJson, createdAt: Date.now(), }); useMediaGenerationStore.getState().markDone(req.elementId, result.ossUrl); return; } // Fallback: fetch blob via proxy-media throwIfAborted(abortSignal); const blob = await fetchAsBlob(result.url); await db.mediaFiles.put({ id: mediaFileKey(stageId, req.elementId), stageId, type: 'image', blob, mimeType: 'image/png', size: blob.size, prompt: req.prompt, params: paramsJson, createdAt: Date.now(), }); const objectUrl = URL.createObjectURL(blob); useMediaGenerationStore.getState().markDone(req.elementId, objectUrl); } else { const result = await callVideoApi(req, abortSignal); // CDN path: server already uploaded to OSS if (result.ossUrl) { throwIfAborted(abortSignal); await db.mediaFiles.put({ id: mediaFileKey(stageId, req.elementId), stageId, type: 'video', blob: new Blob([]), mimeType: 'video/mp4', size: 0, ossKey: result.ossUrl, posterOssKey: result.posterOssUrl, prompt: req.prompt, params: paramsJson, createdAt: Date.now(), }); useMediaGenerationStore .getState() .markDone(req.elementId, result.ossUrl, result.posterOssUrl); return; } // Fallback: fetch blob via proxy-media throwIfAborted(abortSignal); const blob = await fetchAsBlob(result.url); const posterBlob = result.poster ? await fetchAsBlob(result.poster).catch(() => undefined) : undefined; await db.mediaFiles.put({ id: mediaFileKey(stageId, req.elementId), stageId, type: 'video', blob, mimeType: 'video/mp4', size: blob.size, poster: posterBlob, prompt: req.prompt, params: paramsJson, createdAt: Date.now(), }); const objectUrl = URL.createObjectURL(blob); const posterObjectUrl = posterBlob ? URL.createObjectURL(posterBlob) : undefined; useMediaGenerationStore.getState().markDone(req.elementId, objectUrl, posterObjectUrl); } } catch (err) { if (abortSignal?.aborted) { // A submitted video MaaS task keeps running to a billable terminal state // server-side even after this client stops polling. Mark either media // task retryable instead of leaving it stuck in `generating`; note that // retrying a video submits a second job rather than resuming the first. const abortedMessage = req.type === 'video' ? 'Video generation polling was aborted; retry to submit a new job' : 'Image generation was aborted; retry to submit a new request'; useMediaGenerationStore.getState().markFailed(req.elementId, abortedMessage); return; } const message = err instanceof Error ? err.message : String(err); const errorCode = err instanceof MediaApiError ? err.errorCode : undefined; log.error(`Failed ${req.elementId}:`, message); useMediaGenerationStore.getState().markFailed(req.elementId, message, errorCode); // Persist non-retryable failures to IndexedDB so they survive page refresh if (errorCode) { await db.mediaFiles .put({ id: mediaFileKey(stageId, req.elementId), stageId, type: req.type, blob: new Blob(), // empty placeholder mimeType: req.type === 'image' ? 'image/png' : 'video/mp4', size: 0, prompt: req.prompt, params: JSON.stringify({ aspectRatio: req.aspectRatio, style: req.style }), error: message, errorCode, createdAt: Date.now(), }) .catch(() => {}); // best-effort } } } async function callImageApi( req: MediaGenerationRequest, stageId: string, abortSignal?: AbortSignal, ): Promise<{ url: string; ossUrl?: string }> { const settings = useSettingsStore.getState(); const providerConfig = settings.imageProvidersConfig?.[settings.imageProviderId]; const response = await fetch('/api/generate/image', { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-image-provider': settings.imageProviderId || '', 'x-image-model': settings.imageModelId || '', 'x-api-key': providerConfig?.apiKey || '', 'x-base-url': providerConfig?.baseUrl || '', }, body: JSON.stringify({ prompt: req.prompt, aspectRatio: req.aspectRatio, style: req.style, stageId, }), signal: abortSignal, }); if (!response.ok) { const data = await response.json().catch(() => ({})); throw new MediaApiError(data.error || `Image API returned ${response.status}`, data.errorCode); } const data = await response.json(); if (!data.success) throw new MediaApiError(data.error || 'Image generation failed', data.errorCode); // Result may have ossUrl (CDN direct), url, or base64 const ossUrl = data.result?.ossUrl as string | undefined; const url = data.result?.url || (data.result?.base64 ? `data:image/png;base64,${data.result.base64}` : ''); if (!ossUrl && !url) throw new Error('No image URL in response'); return { url, ossUrl }; } async function callVideoApi( req: MediaGenerationRequest, abortSignal?: AbortSignal, ): Promise<{ url: string; poster?: string; ossUrl?: string; posterOssUrl?: string; width?: number; height?: number; duration?: number; }> { const settings = useSettingsStore.getState(); const providerConfig = settings.videoProvidersConfig?.[settings.videoProviderId]; const response = await fetch('/api/generate/video', { method: 'POST', headers: { 'Content-Type': 'application/json', 'x-video-provider': settings.videoProviderId || '', 'x-video-model': settings.videoModelId || '', 'x-api-key': providerConfig?.apiKey || '', 'x-base-url': providerConfig?.baseUrl || '', }, body: JSON.stringify({ prompt: req.prompt, aspectRatio: req.aspectRatio, }), signal: abortSignal, }); if (!response.ok) { const data = await response.json().catch(() => ({})); throw new MediaApiError(data.error || `Video API returned ${response.status}`, data.errorCode); } const data = await response.json(); if (!data.success) throw new MediaApiError(data.error || 'Video generation failed', data.errorCode); const url = data.result?.url; if (!url) throw new Error('No video URL in response'); return { url, poster: data.result?.poster, ossUrl: data.result?.ossUrl, posterOssUrl: data.result?.posterOssUrl, width: data.result?.width, height: data.result?.height, duration: data.result?.duration, }; } async function fetchAsBlob(url: string): Promise { // For data URLs, convert directly if (url.startsWith('data:')) { const res = await fetch(url); return res.blob(); } // For remote URLs, proxy through our server to bypass CORS restrictions. // Routed through the shared proxy-media negative cache so a permanently // failed URL (4xx) is not re-fetched by retries or later generation passes. if (url.startsWith('http://') || url.startsWith('https://')) { const res = await fetchProxiedMediaUrl(url); if (!res.ok) { const data = await res.json().catch(() => ({})); throw new Error(data.error || `Proxy fetch failed: ${res.status}`); } return res.blob(); } // Relative URLs (shouldn't happen, but handle gracefully) const res = await fetch(url); if (!res.ok) throw new Error(`Failed to fetch blob: ${res.status}`); return res.blob(); }