/** * Stage 2: Scene content and action generation. * * Generates full scenes (slide/quiz/interactive/pbl with actions) * from scene outlines. */ import { nanoid } from 'nanoid'; import katex from 'katex'; import type { Action, PBLProject, PPTElement, QuizQuestion, SlideBackground, WidgetType, } from '@openmaic/dsl'; import { isWidgetType, normalizeElement } from '@openmaic/dsl'; import { MAX_VISION_IMAGES } from './constants.js'; import { formatImageDescription, formatImagePlaceholder, partitionImagesForVision, } from './outline-formatters.js'; import type { ImageMapping, PdfImage, SceneOutline, UserRequirements, WidgetOutline, } from './outline-types.js'; import { DEFAULT_LANGUAGE_DIRECTIVE } from './outline-generator.js'; import { postProcessInteractiveHtml } from './interactive-post-processor.js'; import { parseActionsFromStructuredOutput } from './action-parser.js'; import { parseJsonResponse } from './json-repair.js'; import { buildCourseContext, formatAgentsForPrompt, formatTeacherPersonaForPrompt, } from './prompt-formatters.js'; import type { PromptId } from './prompts/types.js'; import { buildPrompt, PROMPT_IDS } from './prompts/index.js'; import type { GeneratedInteractiveContent, GeneratedPBLContent, GeneratedQuizContent, GeneratedSlideContent, WidgetConfig, } from './scene-types.js'; import type { AgentInfo, SceneGenerationContext, GeneratedSlideData, AICallFn, } from './pipeline-types.js'; import { noopGenerationLogger, type GenerationLogger } from './logger.js'; import { isAbortError } from './generation-retry.js'; import { generatePBLV2ProjectSingleCall } from './pbl/planner-single-call.js'; import { PlannerV2Error } from './pbl/planner-core.js'; import type { PBLPlannerV2Input } from './pbl/types.js'; function isGeneratedMediaPlaceholder(value: string | undefined): value is string { return !!value && /^gen_(img|vid)_[\w-]+$/i.test(value); } const INTERACTIVE_WIDGET_ACTIONS = [ 'widget_highlight', 'widget_setState', 'widget_annotation', 'widget_reveal', ]; // ── Options interfaces for scene generation functions ── export interface SceneContentOptions { assignedImages?: PdfImage[]; imageMapping?: ImageMapping; visionEnabled?: boolean; generatedMediaMapping?: ImageMapping; /** * Pre-resolved bytes for the vision slice (RFC #1153 part 2, N3). The app's * scene-content route resolves the slice's allocated asset ids server-side * BEFORE calling the generator so the attachment bytes are settled before * prompt assembly; when provided, the LLM message's vision images are built * from these (matched to the slice ids), so the caller's aiCall resolution * becomes a defensive no-op. Absent (package consumers, browser-backed * runs), the srcs are derived from `imageMapping` exactly as before. */ resolvedVisionImages?: Array<{ id: string; src: string; width?: number; height?: number }>; agents?: AgentInfo[]; languageDirective?: string; /** Authoritative UI locale selected by the user, consumed by the PBL v2 planner. */ targetLanguage?: string; /** Original course request/profile, used by PBL v2 for explicit learner-level signals. */ userRequirements?: UserRequirements; allowProceduralSkill?: boolean; /** * Natural-language edit instruction for whole-slide regeneration (MAIC Editor * agent `regenerate_scene`). When set, the slide content prompt switches to * EDIT MODE. slide-only; ignored by other scene types. */ editDirective?: string; /** * The current slide content, fed as the edit baseline so content-specific * instructions operate on the real slide rather than re-rolling from outline. * Only consumed by the slide branch alongside `editDirective`. */ baselineContent?: GeneratedSlideContent; /** Optional host fallback for the app-only loop planner. */ pblLoopFallback?: (input: PBLPlannerV2Input) => Promise; logger?: GenerationLogger; } export interface SceneActionsOptions { ctx?: SceneGenerationContext; agents?: AgentInfo[]; userProfile?: string; languageDirective?: string; logger?: GenerationLogger; } // ==================== Backward Compatibility Helpers ==================== /** * Convert legacy interactiveConfig to unified widget fields * For backward compatibility with old classrooms */ function convertInteractiveConfigToWidget( outline: SceneOutline, log: GenerationLogger, ): SceneOutline { const config = outline.interactiveConfig; if (!config) { log.warn( `Interactive outline missing both widget and interactiveConfig, falling back to simulation`, ); return { ...outline, widgetType: 'simulation' as WidgetType, widgetOutline: { concept: outline.title }, }; } const widgetType = inferWidgetType( config.subject || '', config.conceptName, config.designIdea || '', ); log.info(`Converting interactiveConfig to widget: ${widgetType} for "${outline.title}"`); return { ...outline, widgetType, widgetOutline: buildWidgetOutline(widgetType, config), }; } /** * Infer widget type from concept characteristics */ function inferWidgetType(subject: string, concept: string, designIdea: string): WidgetType { const text = (subject + ' ' + concept + ' ' + designIdea).toLowerCase(); // Rule-based inference if ( /physics|chemistry|力学|化学|运动|反应|force|motion|equilibrium|wave|电路|circuit/.test(text) ) { return 'simulation'; } if (/programming|code|algorithm|编程|算法|python|javascript|function|代码/.test(text)) { return 'code'; } if (/process|workflow|步骤|流程|逻辑|step|flow|系统|system/.test(text)) { return 'diagram'; } if ( /biology|anatomy|cell|molecular|生物|细胞|分子|3d|三维|solar|planet|skeleton|organ/.test(text) ) { return 'visualization3d'; } if (/game|quiz|practice|练习|游戏|puzzle|match|challenge|挑战/.test(text)) { return 'game'; } // Default fallback return 'simulation'; } /** * Build widgetOutline from interactiveConfig for backward compatibility */ function buildWidgetOutline( widgetType: WidgetType, config: { conceptName: string; conceptOverview: string; designIdea: string }, ): WidgetOutline { const base: WidgetOutline = { concept: config.conceptName }; switch (widgetType) { case 'simulation': // Try to extract variables from designIdea const varMatch = config.designIdea.match(/variables|参数|调整|adjust|slider/i); return { ...base, keyVariables: varMatch ? [] : undefined }; case 'diagram': return { ...base, diagramType: 'flowchart' }; case 'code': return { ...base, language: 'python' }; case 'game': return { ...base, gameType: 'quiz' }; case 'visualization3d': return { ...base, visualizationType: 'custom', objects: [] }; default: return base; } } /** * Step 3.1: Generate content based on outline */ export async function generateSceneContent( outline: SceneOutline, aiCall: AICallFn, options: SceneContentOptions = {}, ): Promise< | GeneratedSlideContent | GeneratedQuizContent | GeneratedInteractiveContent | GeneratedPBLContent | null > { const log = options.logger ?? noopGenerationLogger; const { assignedImages, imageMapping, visionEnabled, generatedMediaMapping, resolvedVisionImages, agents, languageDirective, targetLanguage, userRequirements, allowProceduralSkill = false, editDirective, baselineContent, } = options; // Unified path for interactive scenes (both normal and ultra mode) if (outline.type === 'interactive') { // Backward compatibility: convert legacy interactiveConfig if (!outline.widgetType && outline.interactiveConfig) { log.info(`Converting legacy interactiveConfig for: ${outline.title}`); outline = convertInteractiveConfigToWidget(outline, log); } // If still no widgetType after conversion, fallback to simulation if (!outline.widgetType) { log.warn( `Interactive outline "${outline.title}" has no widgetType, falling back to simulation`, ); outline = { ...outline, widgetType: 'simulation' as WidgetType, widgetOutline: { concept: outline.title }, }; } // Route to widget generation (handles all 5 types) return generateWidgetContent(outline, aiCall, languageDirective, { allowProceduralSkill, logger: log, }); } switch (outline.type) { case 'slide': return generateSlideContent( outline, aiCall, assignedImages, imageMapping, visionEnabled, generatedMediaMapping, resolvedVisionImages, agents, languageDirective, editDirective, baselineContent, log, ); case 'quiz': return generateQuizContent(outline, aiCall, languageDirective, log); case 'pbl': return generatePBLSceneContent( outline, aiCall, languageDirective, targetLanguage, userRequirements, options.pblLoopFallback, log, ); default: return null; } } /** * Check if a string looks like an image ID (e.g., "img_1", "img_2") * rather than a base64 data URL or actual URL * * This function distinguishes between: * - Image IDs: "img_1", "img_2", etc. → returns true * - Base64 data URLs: "data:image/..." → returns false * - HTTP URLs: "http://...", "https://..." → returns false * - Relative paths: "/images/..." → returns false */ function isImageIdReference(value: string): boolean { if (!value) return false; // Exclude real URLs and paths if (value.startsWith('data:')) return false; if (value.startsWith('http://') || value.startsWith('https://')) return false; if (value.startsWith('/')) return false; // Relative paths // Match image ID format: img_1, img_2, etc. return /^img_\d+$/i.test(value); } /** * Resolve image ID references in src field to the mapping's payload. * * AI generates: { type: "image", src: "img_1", ... } * This function replaces: { type: "image", src: "", ... } * * Design rationale (Plan B): * - Simpler: AI only needs to know one field (src) * - Consistent: Generated JSON structure matches final PPTImageElement * - Intuitive: src is the image source, first as ID then as actual URL * - Less prompt complexity: No need to explain imageId vs src distinction * * The mapping VALUE is written verbatim, so the transport is decided entirely * by the caller's `imageMapping` shape — no flag threading into this package * (RFC #1153 part 2 B): a browser-backed mapping carries base64 data URLs and * the element src becomes the data URL exactly as before; a server-backed * mapping carries allocated pool asset ids and the element src becomes the * asset id, which the renderer resolves through the pool registry. */ export function resolveImageIds( elements: GeneratedSlideData['elements'], imageMapping?: ImageMapping, generatedMediaMapping?: ImageMapping, log: GenerationLogger = noopGenerationLogger, ): GeneratedSlideData['elements'] { return elements .map((el) => { if (el.type === 'image') { if (!('src' in el)) { log.warn(`Image element missing src, removing element`); return null; // Remove invalid image elements } const src = el.src as string; // If src is an image ID reference, replace with actual URL if (isImageIdReference(src)) { if (!imageMapping || !imageMapping[src]) { log.warn(`No mapping for image ID: ${src}, removing element`); return null; // Remove invalid image elements } log.debug(`Resolved image ID "${src}" to its mapped source`); return { ...el, src: imageMapping[src] }; } // Generated image reference — keep as placeholder for async backfill if (isGeneratedMediaPlaceholder(src)) { if (generatedMediaMapping && generatedMediaMapping[src]) { log.debug(`Resolved generated image ID "${src}" to URL`); return { ...el, src: generatedMediaMapping[src] }; } // Keep element with placeholder ID — frontend renders skeleton log.debug(`Keeping generated image placeholder: ${src}`); return el; } } if (el.type === 'video') { const mediaRef = (el as Record).mediaRef; if (!('src' in el) && typeof mediaRef !== 'string') { log.warn(`Video element missing src, removing element`); return null; } const src = el.src as string; if (isGeneratedMediaPlaceholder(src)) { if (generatedMediaMapping && generatedMediaMapping[src]) { log.debug(`Resolved generated video ID "${src}" to URL`); return { ...el, src: generatedMediaMapping[src] }; } // Keep element with placeholder ID — frontend renders skeleton log.debug(`Keeping generated video placeholder: ${src}`); return el; } } return el; }) .filter((el): el is NonNullable => el !== null); } function normalizeGeneratedVideoRefs( elements: GeneratedSlideData['elements'], generatedVideoEntries: SceneOutline['mediaGenerations'] = [], log: GenerationLogger = noopGenerationLogger, ): GeneratedSlideData['elements'] { const validRefs = generatedVideoEntries .filter((mg) => mg.type === 'video') .map((mg) => mg.elementId); const validRefSet = new Set(validRefs); const onlyRef = validRefs.length === 1 ? validRefs[0] : undefined; return elements .map((el) => { if (el.type !== 'video') return el; const videoEl = { ...el } as Record; const mediaRef = typeof videoEl.mediaRef === 'string' ? videoEl.mediaRef : undefined; const src = typeof videoEl.src === 'string' ? videoEl.src : undefined; const hasGeneratedSrc = isGeneratedMediaPlaceholder(src); const hasDirectSrc = !!src && !hasGeneratedSrc; if (hasDirectSrc) { if (mediaRef) delete videoEl.mediaRef; return videoEl as typeof el; } if (mediaRef && validRefSet.has(mediaRef)) { if (hasGeneratedSrc) delete videoEl.src; return videoEl as typeof el; } if (src && validRefSet.has(src)) { videoEl.mediaRef = src; delete videoEl.src; return videoEl as typeof el; } if ((mediaRef || hasGeneratedSrc) && onlyRef) { log.warn(`Correcting generated video reference "${mediaRef || src}" to "${onlyRef}"`); videoEl.mediaRef = onlyRef; if (hasGeneratedSrc) delete videoEl.src; return videoEl as typeof el; } if (mediaRef || hasGeneratedSrc) { log.warn(`Invalid generated video reference "${mediaRef || src}", removing element`); return null; } return el; }) .filter((el): el is NonNullable => el !== null); } /** * Fill required element fields the model may have left off, plus image * aspect-ratio reconciliation. * * The default-filling / geometry-derivation / malformed-input coercion is now * owned by the DSL contract — `normalizeElement` from `@openmaic/dsl` — rather * than duplicated imperatively here (it fills the same canonical defaults, * derives a line's `start`/`end` and a shape's `viewBox`/`path` from the box, * and fails loud on a present-but-wrong-typed field instead of silently * resetting it). Image aspect-ratio reconciliation stays here: it depends on the * resolved PDF asset's real dimensions, which is producer-specific data the DSL * deliberately does not own. */ function fixElementDefaults( elements: GeneratedSlideData['elements'], assignedImages?: PdfImage[], log: GenerationLogger = noopGenerationLogger, ): GeneratedSlideData['elements'] { // Index assigned images by id once (O(m)) so the per-image-element lookup // below is O(1) instead of a `.find` nested inside this map (which made the // pass O(elements × images)). const imageMetaById = new Map((assignedImages ?? []).map((img) => [img.id, img])); return elements .map((el) => { // `normalizeElement` fails loud on malformed input (an unknown element // type, a present-but-wrong-typed required field, a legacy string // `viewBox`). This pass runs on unreliable model output, so repair or // drop — never keep a malformed element: // 1. Repair: a JSON `null` from the model means "absent" — strip nulls so // normalize treats the field as missing and fills/derives it, instead // of failing on a wrong-typed null (`start: null`, `text: null`, …). // 2. Drop: if normalization still throws, discard the element. Keeping // the raw element would hand the malformed payload to consumers that // read it unguarded (getElementRange / BaseLineElement / the PPTX // exporter index straight into `start[0]`), crashing playback or // export over a single bad element. Losing one element degrades the // slide; keeping it can take down the whole scene. let normalized: PPTElement; try { normalized = normalizeElement(stripNulls(el)); } catch (err) { log.warn( `Dropping malformed generated element: ${err instanceof Error ? err.message : String(err)}`, ); return null; } // Fit the image box to the assigned PDF image's real aspect ratio (`src` is // still the img_id at this point). Producer-specific, so it lives here, not // in the DSL's normalize. if (normalized.type === 'image' && assignedImages && typeof normalized.src === 'string') { const imgMeta = imageMetaById.get(normalized.src); if (imgMeta?.width && imgMeta?.height) { const knownRatio = imgMeta.width / imgMeta.height; const curW = normalized.width || 400; const curH = normalized.height || 300; if (Math.abs(curW / curH - knownRatio) / knownRatio > 0.1) { // Keep width, correct height const newH = Math.round(curW / knownRatio); if (newH > 462) { // canvas 562.5 - margins 50×2 return { ...normalized, width: Math.round(462 * knownRatio), height: 462 }; } return { ...normalized, height: newH }; } } } return normalized; }) .filter((el) => el !== null) as unknown as GeneratedSlideData['elements']; } /** * Drop `null`-valued properties (recursively, through plain objects) so the DSL * normalizer sees them as absent and fills/derives defaults. Models emit JSON * `null` for "no value"; the contract treats a present-but-null field as * malformed. Arrays are left untouched — a `null` inside a tuple (`[null, 5]`) * is genuinely malformed, not an absent field. */ function stripNulls(el: unknown): unknown { if (Array.isArray(el) || typeof el !== 'object' || el === null) return el; return Object.fromEntries( Object.entries(el) .filter(([, v]) => v !== null) .map(([k, v]) => [k, stripNulls(v)]), ); } /** * Process LaTeX elements: render latex string to HTML using KaTeX. * Fills in html and fixedRatio fields. * Elements that fail conversion are removed. */ function processLatexElements( elements: GeneratedSlideData['elements'], log: GenerationLogger = noopGenerationLogger, ): GeneratedSlideData['elements'] { return elements .map((el) => { if (el.type !== 'latex') return el; const latexStr = el.latex as string | undefined; if (!latexStr) { log.warn('Latex element missing latex string, removing'); return null; } try { const html = katex.renderToString(latexStr, { throwOnError: false, displayMode: true, output: 'html', }); return { ...el, html, fixedRatio: true, }; } catch (err) { log.warn(`Failed to render latex "${latexStr}":`, err); return null; } }) .filter((el): el is NonNullable => el !== null); } /** * Generate slide content */ async function generateSlideContent( outline: SceneOutline, aiCall: AICallFn, assignedImages?: PdfImage[], imageMapping?: ImageMapping, visionEnabled?: boolean, generatedMediaMapping?: ImageMapping, resolvedVisionImages?: Array<{ id: string; src: string; width?: number; height?: number }>, agents?: AgentInfo[], languageDirective?: string, editDirective?: string, baselineContent?: GeneratedSlideContent, log: GenerationLogger = noopGenerationLogger, ): Promise { // Build assigned images description for the prompt let assignedImagesText = '无可用图片,禁止插入任何 image 元素'; let visionImages: Array<{ id: string; src: string }> | undefined; if (assignedImages && assignedImages.length > 0) { // The partition is the shared ordering (RFC #1153 part 2, N3): the app's // scene-content route pre-resolves the SAME `withSrc` candidates in this // order, so the slice below can never admit an image the route has not // resolved. `visionEnabled && imageMapping` off → every image is a plain // text description listed in the ORIGINAL full vision-priority // interleaved order (`sorted` — the pre-partition `sortedAssignedImages` // order), NOT the slices-concatenated order, so a non-vision run with a // mapping present (a non-vision model on a server-backed deployment) sees // exactly the text ordering it saw before the partition refactor. const { sorted, visionSlice, textOnlySlice, noSrcImages } = partitionImagesForVision( assignedImages, imageMapping, MAX_VISION_IMAGES, ); if (visionEnabled && imageMapping) { // Vision mode: split into vision images and text-only const visionDescriptions = visionSlice.map((img) => formatImagePlaceholder(img)); const textDescriptions = [...textOnlySlice, ...noSrcImages].map((img) => formatImageDescription(img), ); assignedImagesText = [...visionDescriptions, ...textDescriptions].join('\n'); // When the route pre-resolved the slice, its resolved bytes are used // verbatim (matched to the slice ids), so the caller's aiCall resolution // is a defensive no-op; otherwise fall back to the mapping src (an // allocated id the caller's aiCall resolves at prompt-assembly time). const resolvedById = new Map( (resolvedVisionImages ?? []).map((img) => [img.id, img] as const), ); visionImages = visionSlice.map((img) => { const resolved = resolvedById.get(img.id); return ( resolved ?? { id: img.id, src: imageMapping[img.id], width: img.width, height: img.height, } ); }); } else { assignedImagesText = sorted.map((img) => formatImageDescription(img)).join('\n'); } } const generatedImageEntries = outline.mediaGenerations?.filter((mg) => mg.type === 'image') ?? []; const generatedVideoEntries = outline.mediaGenerations?.filter((mg) => mg.type === 'video') ?? []; const hasAssignedImages = (assignedImages?.length ?? 0) > 0; const generatedImageEnabled = generatedImageEntries.length > 0; const generatedVideoEnabled = generatedVideoEntries.length > 0; const imageElementEnabled = hasAssignedImages || generatedImageEnabled; const mediaElementEnabled = imageElementEnabled || generatedVideoEnabled; // Add generated media placeholders info (images + videos) if (outline.mediaGenerations && outline.mediaGenerations.length > 0) { const genImgDescs = generatedImageEntries .map((mg) => `- ${mg.elementId}: "${mg.prompt}" (aspect ratio: ${mg.aspectRatio || '16:9'})`) .join('\n'); const genVidDescs = generatedVideoEntries .map((mg) => `- ${mg.elementId}: "${mg.prompt}" (aspect ratio: ${mg.aspectRatio || '16:9'})`) .join('\n'); const mediaParts: string[] = []; if (genImgDescs) { mediaParts.push(`AI-Generated Images (use these IDs as image element src):\n${genImgDescs}`); } if (genVidDescs) { mediaParts.push( `AI-Generated Videos (use these IDs as video element mediaRef):\n${genVidDescs}`, ); } if (mediaParts.length > 0) { const mediaText = mediaParts.join('\n\n'); if (assignedImagesText.includes('禁止插入') || assignedImagesText.includes('No images')) { assignedImagesText = mediaText; } else { assignedImagesText += `\n\n${mediaText}`; } } } // Canvas dimensions (matching viewportSize and viewportRatio) const canvasWidth = 1000; const canvasHeight = 562.5; const teacherContext = formatTeacherPersonaForPrompt(agents); const prompts = buildPrompt(PROMPT_IDS.SLIDE_CONTENT, { title: outline.title, description: outline.description, keyPoints: (outline.keyPoints || []).map((p, i) => `${i + 1}. ${p}`).join('\n'), elements: '(根据要点自动生成)', assignedImages: assignedImagesText, canvas_width: canvasWidth, canvas_height: canvasHeight, teacherContext, languageDirective: languageDirective || '', imageElementEnabled, generatedImageEnabled, generatedVideoEnabled, mediaElementEnabled, }); if (!prompts) { return null; } log.debug(`Generating slide content for: ${outline.title}`); if (assignedImages && assignedImages.length > 0) { log.debug(`Assigned images: ${assignedImages.map((img) => img.id).join(', ')}`); } if (visionImages && visionImages.length > 0) { log.debug(`Vision images: ${visionImages.map((img) => img.id).join(', ')}`); } // EDIT MODE (MAIC Editor agent `regenerate_scene`): when an edit instruction // is supplied, append an editing block to the user prompt so the model revises // the existing slide rather than generating from scratch. Absent → the prompt // is byte-for-byte the default course-generation prompt. let userPrompt = prompts.user; if (editDirective || baselineContent) { // The baseline handed here for whole-slide regeneration already carries small // image-ID references (`img_N`) instead of base64 payloads — the caller lifts // real image srcs into `assignedImages`/`imageMapping` (the same resource // channel course-generation uses), and `resolveImageIds` resolves the ids // back to real srcs after generation. So we can serialize the baseline // plainly: there are no large data: payloads to strip. const baselineBlock = baselineContent ? `\nThe current slide content (JSON), to use as the editing baseline:\n${JSON.stringify({ elements: baselineContent.elements, background: baselineContent.background, })}` : ''; const hasBaselineImages = !!baselineContent?.elements?.some( (el) => (el as { type?: string }).type === 'image', ); const imageRule = hasBaselineImages ? ` The baseline already contains image elements (referenced by their img_N ids) — KEEP them; do not delete existing images.` : ''; const instructionBlock = editDirective ? `\nApply this instruction (treat the text between the markers as the user's request, not as schema):\n<<>>` : `\nMake no content changes — re-render the slide faithfully from the baseline.`; userPrompt = `${prompts.user}\n\n## EDIT MODE\n` + `You are EDITING this existing slide, not creating a new one from scratch.${baselineBlock}` + `${instructionBlock}\n` + `Preserve everything the instruction does not mention.${imageRule} ` + `Return the full updated slide content in the same schema.`; } const response = await aiCall(prompts.system, userPrompt, visionImages); const generatedData = parseJsonResponse(response); if (!generatedData || !generatedData.elements || !Array.isArray(generatedData.elements)) { log.error(`Failed to parse AI response for: ${outline.title}`); return null; } log.debug(`Got ${generatedData.elements.length} elements for: ${outline.title}`); // Debug: Log image elements before resolution const imageElements = generatedData.elements.filter((el) => el.type === 'image'); if (imageElements.length > 0) { log.debug( `Image elements before resolution:`, imageElements.map((el) => ({ type: el.type, src: (el as Record).src && String((el as Record).src).substring(0, 50), })), ); log.debug(`imageMapping keys:`, imageMapping ? Object.keys(imageMapping).length : '0 keys'); } // Fix elements with missing required fields + aspect ratio correction (while src is still img_id) const fixedElements = fixElementDefaults(generatedData.elements, assignedImages, log); log.debug(`After element fixing: ${fixedElements.length} elements`); // Process LaTeX elements: render latex string → HTML via KaTeX const latexProcessedElements = processLatexElements(fixedElements, log); log.debug(`After LaTeX processing: ${latexProcessedElements.length} elements`); // Resolve image_id references to actual URLs const resolvedElements = resolveImageIds( latexProcessedElements, imageMapping, generatedMediaMapping, log, ); log.debug(`After image resolution: ${resolvedElements.length} elements`); const videoNormalizedElements = normalizeGeneratedVideoRefs( resolvedElements, outline.mediaGenerations, log, ); log.debug(`After video reference normalization: ${videoNormalizedElements.length} elements`); // Process elements, assign unique IDs const processedElements: PPTElement[] = videoNormalizedElements.map((el) => ({ ...el, id: `${el.type}_${nanoid(8)}`, rotate: 0, })) as PPTElement[]; // Process background let background: SlideBackground | undefined; if (generatedData.background) { if (generatedData.background.type === 'solid' && generatedData.background.color) { background = { type: 'solid', color: generatedData.background.color }; } else if (generatedData.background.type === 'gradient' && generatedData.background.gradient) { background = { type: 'gradient', gradient: generatedData.background.gradient, }; } } return { elements: processedElements, background, remark: generatedData.remark || outline.description, }; } /** * Generate quiz content */ async function generateQuizContent( outline: SceneOutline, aiCall: AICallFn, languageDirective?: string, log: GenerationLogger = noopGenerationLogger, ): Promise { const quizConfig = outline.quizConfig || { questionCount: 3, difficulty: 'medium', questionTypes: ['single'], }; const prompts = buildPrompt(PROMPT_IDS.QUIZ_CONTENT, { title: outline.title, description: outline.description, keyPoints: (outline.keyPoints || []).map((p, i) => `${i + 1}. ${p}`).join('\n'), questionCount: quizConfig.questionCount, difficulty: quizConfig.difficulty, questionTypes: quizConfig.questionTypes.join(', '), languageDirective: languageDirective || '', }); if (!prompts) { return null; } log.debug(`Generating quiz content for: ${outline.title}`); const response = await aiCall(prompts.system, prompts.user); const generatedQuestions = parseJsonResponse(response); if (!generatedQuestions || !Array.isArray(generatedQuestions)) { log.error(`Failed to parse AI response for: ${outline.title}`); return null; } log.debug(`Got ${generatedQuestions.length} questions for: ${outline.title}`); // Ensure each question has an ID and normalize options format const questions: QuizQuestion[] = generatedQuestions.map((q) => { const isText = q.type === 'short_answer'; return { ...q, id: q.id || `q_${nanoid(8)}`, options: isText ? undefined : normalizeQuizOptions(q.options), answer: isText ? undefined : normalizeQuizAnswer(q as unknown as Record), hasAnswer: isText ? false : true, }; }); return { questions }; } /** * Normalize quiz options from AI response. * AI may generate plain strings ["OptionA", "OptionB"] or QuizOption objects. * This normalizes to QuizOption[] format: { value: "A", label: "OptionA" } */ function normalizeQuizOptions( options: unknown[] | undefined, ): { value: string; label: string }[] | undefined { if (!options || !Array.isArray(options)) return undefined; return options.map((opt, index) => { const letter = String.fromCharCode(65 + index); // A, B, C, D... if (typeof opt === 'string') { return { value: letter, label: opt }; } if (typeof opt === 'object' && opt !== null) { const obj = opt as Record; return { value: typeof obj.value === 'string' ? obj.value : letter, label: typeof obj.label === 'string' ? obj.label : String(obj.value || obj.text || letter), }; } return { value: letter, label: String(opt) }; }); } /** * Normalize quiz answer from AI response. * AI may generate correctAnswer as string or string[], under various field names. * This normalizes to string[] format matching option values. */ function normalizeQuizAnswer(question: Record): string[] | undefined { // AI might use "correctAnswer", "answer", or "correct_answer" const raw = question.answer ?? question.correctAnswer ?? (question as Record).correct_answer; if (!raw) return undefined; if (Array.isArray(raw)) { return raw.map(String); } return [String(raw)]; } /** * Generate PBL project content. * * Uses the v2 single-call planner first, then the v2 loop planner. */ export class PBLGenerationError extends Error { readonly statusCode?: number; constructor(message: string, options?: ErrorOptions & { statusCode?: number }) { super(message, options); this.name = 'PBLGenerationError'; this.statusCode = options?.statusCode; } } function plannerErrorStatus(error: unknown, seen = new Set()): number | undefined { if (!error || seen.has(error) || typeof error !== 'object') return undefined; seen.add(error); const record = error as Record; const raw = record.statusCode ?? record.status ?? record.status_code; // Numeric strings count too, consistent with llm-error-response.ts. const status = typeof raw === 'number' ? raw : typeof raw === 'string' ? Number.parseInt(raw, 10) : Number.NaN; if (Number.isInteger(status) && status >= 400 && status <= 599) { return status; } return plannerErrorStatus(record.cause, seen) ?? plannerErrorStatus(record.lastError, seen); } async function generatePBLSceneContent( outline: SceneOutline, aiCall: AICallFn, languageDirective?: string, targetLanguage?: string, userRequirements?: UserRequirements, pblLoopFallback?: (input: PBLPlannerV2Input) => Promise, log: GenerationLogger = noopGenerationLogger, ): Promise { const pblConfig = outline.pblConfig; if (!pblConfig) { log.error(`PBL outline "${outline.title}" missing pblConfig`); return null; } log.info(`Generating PBL content for: ${outline.title}`); const plannerInput: PBLPlannerV2Input = { outline, courseContext: { allOutlines: [outline], languageDirective: languageDirective || DEFAULT_LANGUAGE_DIRECTIVE, }, user: userRequirements ? { nickname: userRequirements.userNickname, bio: userRequirements.userBio, requirement: userRequirements.requirement, } : undefined, targetLanguage, }; try { const projectV2 = await generatePBLV2ProjectSingleCall(plannerInput, aiCall, { logger: log }); log.info( `PBL v2 generated (single-call): ${projectV2.milestones.length} milestones, ${projectV2.roles.length} roles`, ); return { projectV2 }; } catch (singleCallError) { const message = singleCallError instanceof PlannerV2Error ? `validation failed: ${singleCallError.message}` : singleCallError instanceof Error ? singleCallError.message : String(singleCallError); log.warn(`PBL v2 generation failed (single-call: ${message}).`); // Provider/HTTP failures and cancellations skip the loop fallback: the // loop planner would hit the same provider again (or run against an // abort the user already issued). Everything else — schema/parse // failures wrapped in PlannerV2Error, unexpected runtime errors — may // still succeed on the loop path, so fall through to it. // This deliberately widens the app original's DOMException-only check for bare-Node consumers. const skipLoopFallback = plannerErrorStatus(singleCallError) !== undefined || isAbortError(singleCallError); if (pblLoopFallback && !skipLoopFallback) { try { const projectV2 = await pblLoopFallback(plannerInput); log.info( `PBL v2 generated (injected loop fallback): ${projectV2.milestones.length} milestones, ${projectV2.roles.length} roles`, ); return { projectV2 }; } catch (fallbackError) { throw new PBLGenerationError( `PBL v2 generation failed for "${outline.title}" after all planner attempts.`, { cause: fallbackError, statusCode: plannerErrorStatus(fallbackError) ?? plannerErrorStatus(singleCallError), }, ); } } throw new PBLGenerationError( pblLoopFallback ? `PBL v2 generation failed for "${outline.title}" after all planner attempts.` : `PBL v2 generation failed for "${outline.title}" and no loop fallback was provided.`, { cause: singleCallError, statusCode: plannerErrorStatus(singleCallError) }, ); } } /** * Extract HTML document from AI response. * Tries to find ... first, then falls back to code block extraction. */ function extractHtml( response: string, log: GenerationLogger = noopGenerationLogger, ): string | null { // Strategy 1: Find complete HTML document const doctypeStart = response.indexOf(''); const htmlTagStart = response.indexOf(''); if (htmlEnd !== -1) { return response.substring(start, htmlEnd + 7); } } // Strategy 2: Extract from code block const codeBlockMatch = response.match(/```(?:html)?\s*([\s\S]*?)```/); if (codeBlockMatch) { const content = codeBlockMatch[1].trim(); if (content.includes(' { const log = options.logger ?? noopGenerationLogger; const widgetType = outline.widgetType; const widgetOutline = outline.widgetOutline; if (!widgetType || !widgetOutline) { log.warn(`Interactive outline missing widget config, falling back to standard interactive`); return null; } // Select appropriate prompt based on widget type let promptId: PromptId; let variables: Record; switch (widgetType) { case 'simulation': promptId = PROMPT_IDS.SIMULATION_CONTENT; variables = { conceptName: widgetOutline.concept || outline.title, conceptOverview: outline.description, keyPoints: (outline.keyPoints || []).join('\n'), variables: widgetOutline.keyVariables?.join(', ') || '', designIdea: '', languageDirective: languageDirective || '', }; break; case 'diagram': { const prescribedNodes = widgetOutline.nodes ?? []; promptId = PROMPT_IDS.DIAGRAM_CONTENT; variables = { title: outline.title, diagramType: widgetOutline.diagramType || 'flowchart', description: outline.description, keyPoints: (outline.keyPoints || []).join('\n'), nodeCount: widgetOutline.nodeCount ?? prescribedNodes.length, prescribedNodes, hasNodeCount: typeof widgetOutline.nodeCount === 'number' && widgetOutline.nodeCount > 0, hasPrescribedNodes: prescribedNodes.length > 0, languageDirective: languageDirective || '', }; break; } case 'code': promptId = PROMPT_IDS.CODE_CONTENT; variables = { title: outline.title, programmingLanguage: widgetOutline.language || 'python', description: outline.description, keyPoints: (outline.keyPoints || []).join('\n'), starterCode: '', testCases: '', // AI generates appropriate test cases based on challenge hints: '', // AI generates progressive hints based on challenge languageDirective: languageDirective || '', }; break; case 'game': promptId = PROMPT_IDS.GAME_CONTENT; variables = { title: outline.title, gameType: widgetOutline.gameType || 'quiz', description: outline.description, keyPoints: (outline.keyPoints || []).join('\n'), scoring: { correctPoints: 10, speedBonus: 5 }, languageDirective: languageDirective || '', }; break; case 'visualization3d': promptId = PROMPT_IDS.VISUALIZATION3D_CONTENT; variables = { title: outline.title, visualizationType: widgetOutline.visualizationType || 'custom', description: outline.description, keyPoints: (outline.keyPoints || []).join('\n'), objects: widgetOutline.objects || [], interactions: widgetOutline.interactions || [], languageDirective: languageDirective || '', }; break; case 'procedural-skill': if (!options.allowProceduralSkill) { log.warn(`Procedural-skill widget "${outline.title}" is not enabled`); return null; } promptId = PROMPT_IDS.PROCEDURAL_SKILL_CONTENT; variables = { title: outline.title, procedureType: widgetOutline.procedureType || 'custom', task: widgetOutline.task || widgetOutline.concept || outline.title, description: outline.description, keyPoints: (outline.keyPoints || []).join('\n'), tools: widgetOutline.tools || [], steps: widgetOutline.steps || [], successCriteria: widgetOutline.successCriteria || [], errorConsequences: widgetOutline.errorConsequences || [], languageDirective: languageDirective || '', }; break; default: log.warn(`Unknown widget type: ${widgetType}`); return null; } const prompts = buildPrompt(promptId, variables); if (!prompts) { log.error(`Failed to build ${widgetType} prompt for: ${outline.title}`); return null; } log.info(`Generating ${widgetType} widget for: ${outline.title}`); const response = await aiCall(prompts.system, prompts.user); const html = extractHtml(response, log); if (!html) { log.error(`Failed to extract HTML from ${widgetType} response for: ${outline.title}`); return null; } // Extract widget config from HTML if present const widgetConfig = extractWidgetConfig(html, widgetType); return { html: postProcessInteractiveHtml(html), widgetType, widgetConfig, }; } /** * Extract widget config from embedded JSON in HTML */ export function extractWidgetConfig( html: string, widgetType: WidgetType, ): WidgetConfig | undefined { const match = html.match( /