import { existsSync, readFileSync } from 'node:fs'; import { join } from 'node:path'; import type { Page } from 'playwright'; import { LEAD_IN_S, OUTPUT_DIR } from './config.js'; import { clearVignette, hideAdScene, hideCaption, scrollPaneTo, setCursorScale, showAdScene, showCaption, showOutro, zoomReset, zoomTo, } from './dom.js'; import { fakeType, sleep, smoothDragSliderThumb, smoothMove } from './motion.js'; import { narrationLog } from './narration.js'; import type { Activity, Cue, ScriptCtx, Storyboard, Target } from './script.js'; export interface RunnerResult { /** Wall-clock when the first activity started. */ sceneStartMs: number; /** Wall-clock when the last activity finished (after padding). */ sceneEndMs: number; } const MAP_ZOOM_WHEEL_DELTA = -120; const FALLBACK_MS_PER_WORD = 750; const FALLBACK_TAIL_BUFFER_MS = 800; const CJK_CHARS_PER_FALLBACK_WORD = 2; interface SynthCue { cueIndex: number; text: string; durationMs: number; } /** * Drive the recording from a cue-anchored storyboard. * * Synth runs first and writes ``output/audio/index.json`` with per-cue * measured durations. The runner reads that manifest and sizes each cue's * wall-clock to its measured audio length: ``during`` activities run * sequentially with their declared budgets, then a final wait pads to the * full cue duration so the caption stays on for as long as the audio * plays. ``tail`` activities run after the caption hides; ``gapBeforeMs`` * inserts pure silence before the next cue. * * The activity cursor is wall-clock honest: each step advances it by * ``max(declared, actual)`` so an overrun extends the timeline rather than * silently desyncing the narration manifest from reality. videoTimeMs * recorded for each cue therefore matches the trimmed mp4 frame-for-frame, * which is what the mux step needs to drop audio at the right moment. * * If the audio manifest is missing (``--no-audio`` runs), we fall back to a * worst-case estimate (750ms/word + 800ms buffer) so the visual flow still * works, just without sound. */ export async function runStoryboard( ctx: ScriptCtx, storyboard: Storyboard ): Promise { narrationLog.reset(); const synth = loadSynthIndex(storyboard); const sceneStartMs = Date.now(); const leadInMs = LEAD_IN_S * 1000; const cursor = { ms: 0 }; for (const step of storyboard.pre ?? []) { cursor.ms += await runStep(ctx, step); } for (let i = 0; i < storyboard.cues.length; i++) { await runCue(ctx, storyboard.cues[i], synth[i], cursor, leadInMs); } for (const step of storyboard.post ?? []) { cursor.ms += await runStep(ctx, step); } return { sceneStartMs, sceneEndMs: sceneStartMs + cursor.ms }; } async function runCue( ctx: ScriptCtx, cue: Cue, synth: SynthCue, cursor: { ms: number }, leadInMs: number ): Promise { if (cue.gapBeforeMs > 0) { await sleep(cue.gapBeforeMs); cursor.ms += cue.gapBeforeMs; } const measuredAudioMs = synth.durationMs; narrationLog.add({ text: cue.text, videoTimeMs: cursor.ms + leadInMs, durationMs: measuredAudioMs, }); // The spoken line is never rendered; only an explicit short caption is. if (cue.caption) { await showCaption(ctx.page, cue.caption, cue.captionPlacement); } const during = cue.during ?? []; const declaredSum = during.reduce((s, a) => s + a.durationMs, 0); if (declaredSum > measuredAudioMs + 50) { throw new Error( `Cue ${synth.cueIndex} "${cue.text.slice(0, 40)}…" has ${declaredSum}ms of ` + `during activities but the measured audio is only ${measuredAudioMs}ms. ` + `Trim a during step, lengthen the cue text, or move work into tail.` ); } // Time the during block as a whole: individual steps may overrun their // budgets, but what matters at the cue boundary is total wall-clock. const duringStart = Date.now(); for (const step of during) { await runStep(ctx, step); } const duringElapsed = Date.now() - duringStart; if (duringElapsed < measuredAudioMs) { await sleep(measuredAudioMs - duringElapsed); cursor.ms += measuredAudioMs; } else { cursor.ms += duringElapsed; } if (cue.caption) { await hideCaption(ctx.page); } for (const step of cue.tail ?? []) { cursor.ms += await runStep(ctx, step); } } /** * Run a single activity. Pads short steps to their declared budget, lets * long ones bleed past it, and returns ``max(declared, actual)`` so the * caller can advance the wall-clock-honest cursor. */ async function runStep(ctx: ScriptCtx, step: Activity): Promise { const startedAt = Date.now(); await runActivity(ctx, step); const realMs = Date.now() - startedAt; if (realMs < step.durationMs) { await sleep(step.durationMs - realMs); return step.durationMs; } if (realMs > step.durationMs + 50) { console.log( `[runner] step ${step.kind} ran ${realMs}ms over a ${step.durationMs}ms budget (drift +${realMs - step.durationMs}ms)` ); } return realMs; } async function runActivity(ctx: ScriptCtx, step: Activity): Promise { switch (step.kind) { case 'wait': return; case 'clearVignette': await clearVignette(ctx.page); return; case 'zoomTo': { const focus = await resolveTarget(ctx, step.target); await zoomTo(ctx.page, { scale: step.scale, focusX: focus.x, focusY: focus.y, durationMs: step.durationMs, }); return; } case 'zoomReset': await zoomReset(ctx.page, step.durationMs); return; case 'cursorScale': await setCursorScale(ctx.page, step.scale, step.durationMs); return; case 'moveCursor': { const to = await resolveTarget(ctx, step.target); await smoothMove(ctx.page, ctx.cursor, to, { durationMs: step.durationMs }); ctx.cursor = to; return; } case 'click': { const selectionVersion = ctx.dashboard.getSelectionStatsVersion(); // Hexagon targets are projected from the latest map response; make // sure that response corresponds to the settled viewport (a zoom in // the previous cue may still have postcode fetches in flight). if (step.target.kind === 'hexagon') { await ctx.dashboard.waitForApiIdle(3000); } const candidates = step.target.kind === 'hexagon' && step.waitForSelectionReady ? await ctx.dashboard.visibleHexagonTargets(4) : [await resolveTarget(ctx, step.target)]; let lastError: unknown = null; for (let i = 0; i < candidates.length; i++) { const to = candidates[i]; const moveMs = Math.max(120, Math.round(step.durationMs * 0.7)); await smoothMove(ctx.page, ctx.cursor, to, { durationMs: moveMs }); ctx.cursor = to; await ctx.page.mouse.click(to.x, to.y); if (!step.waitForSelectionReady) return; try { await ctx.dashboard.waitForSelectionReady( selectionVersion, Math.min(step.timeoutMs ?? 12000, i === candidates.length - 1 ? 12000 : 4000) ); return; } catch (err) { lastError = err; } } throw lastError ?? new Error('Click did not open the selection pane'); } case 'clickIfVisible': { const to = await tryResolveTarget(ctx, step.target, step.timeoutMs ?? 700); if (!to) return; const moveMs = Math.max(120, Math.round(step.durationMs * 0.7)); await smoothMove(ctx.page, ctx.cursor, to, { durationMs: moveMs }); ctx.cursor = to; await ctx.page.mouse.click(to.x, to.y); return; } case 'type': await fakeType(ctx.page, step.selector, step.text, step.durationMs); return; case 'mapZoom': { const point = await resolveTarget(ctx, step.target); const mapVersion = ctx.dashboard.getMapDataVersion(); const delta = step.direction === 'out' ? -MAP_ZOOM_WHEEL_DELTA : MAP_ZOOM_WHEEL_DELTA; const handled = await ctx.page.evaluate( async ({ x, y, steps, durationMs, direction, center }) => { const root = document.querySelector('.maplibregl-map') as HTMLElement | null; const fiberKey = root ? Object.getOwnPropertyNames(root).find((key) => key.startsWith('__reactFiber$')) : undefined; let fiber = fiberKey ? (root as unknown as Record)[fiberKey] : null; let mapRef: unknown = null; while (fiber && typeof fiber === 'object') { const maybeFiber = fiber as { ref?: { current?: unknown }; return?: unknown; }; const current = maybeFiber.ref?.current; if ( current && typeof current === 'object' && typeof (current as { getMap?: unknown }).getMap === 'function' ) { mapRef = current; break; } fiber = maybeFiber.return ?? null; } const map = (mapRef as { getMap?: () => unknown } | null)?.getMap?.(); if (!map || typeof map !== 'object') return false; const mapApi = map as { getCanvas: () => HTMLCanvasElement; getZoom: () => number; getMinZoom?: () => number; getMaxZoom?: () => number; unproject: (point: [number, number]) => unknown; zoomTo: ( zoom: number, options: { around?: unknown; duration?: number; essential?: boolean } ) => void; easeTo?: (options: { center?: unknown; zoom?: number; duration?: number; essential?: boolean; }) => void; }; if ( typeof mapApi.getCanvas !== 'function' || typeof mapApi.getZoom !== 'function' || typeof mapApi.unproject !== 'function' || typeof mapApi.zoomTo !== 'function' ) { return false; } const rect = mapApi.getCanvas().getBoundingClientRect(); const around = mapApi.unproject([x - rect.left, y - rect.top]); const sign = direction === 'out' ? -1 : 1; const zoomDelta = Math.max(0.25, Math.min(5.2, steps * 0.28)) * sign; const minZoom = mapApi.getMinZoom?.() ?? 0; const maxZoom = mapApi.getMaxZoom?.() ?? 22; const targetZoom = Math.max(minZoom, Math.min(maxZoom, mapApi.getZoom() + zoomDelta)); if (center && typeof mapApi.easeTo === 'function') { mapApi.easeTo({ center: around, zoom: targetZoom, duration: durationMs, essential: true, }); } else { mapApi.zoomTo(targetZoom, { around, duration: durationMs, essential: true }); } await new Promise((resolve) => window.setTimeout(resolve, durationMs)); return true; }, { x: point.x, y: point.y, steps: step.steps, durationMs: step.durationMs, direction: step.direction, center: step.center ?? false, } ); if (!handled) { const perStepMs = Math.floor(step.durationMs / Math.max(1, step.steps)); await ctx.page.evaluate( async ({ x, y, steps, durationMs, delta }) => { const wait = (ms: number) => new Promise((resolve) => window.setTimeout(resolve, ms)); const perStep = Math.floor(durationMs / Math.max(1, steps)); for (let i = 0; i < steps; i++) { const target = document.elementFromPoint(x, y) ?? document.querySelector('canvas'); target?.dispatchEvent( new WheelEvent('wheel', { bubbles: true, cancelable: true, clientX: x, clientY: y, deltaY: delta, deltaMode: WheelEvent.DOM_DELTA_PIXEL, view: window, }) ); if (perStep > 0) await wait(perStep); } }, { x: point.x, y: point.y, steps: step.steps, durationMs: step.durationMs, delta } ); if (perStepMs > 0) await sleep(0); } if (step.waitForMapSettled) { await ctx.dashboard.waitForMapSettled(mapVersion, step.timeoutMs ?? 12000); } return; } case 'dragSlider': { const mapVersion = ctx.dashboard.getMapDataVersion(); ctx.cursor = await smoothDragSliderThumb( ctx.page, step.thumbSelector, step.trackSelector, ctx.cursor, step.toFraction, step.durationMs ); if (step.waitForMapSettled) { await ctx.dashboard.waitForMapSettled(mapVersion, step.timeoutMs ?? 12000); } return; } case 'submitForm': { const mapVersion = ctx.dashboard.getMapDataVersion(); await ctx.page.evaluate((selector) => { document.querySelector(selector)?.requestSubmit(); }, step.formSelector); if (step.waitForMapSettled) { await ctx.dashboard.waitForMapSettled(mapVersion, step.timeoutMs ?? 12000); } return; } case 'showOutro': await showOutro(ctx.page, step.brand, step.tagline, step.url, step.subtitle); return; case 'showAdScene': await showAdScene(ctx.page, step.scene); return; case 'hideAdScene': await hideAdScene(ctx.page); return; case 'scrollPane': await scrollPaneTo(ctx.page, step.selector, step.top); return; case 'dragSheet': { const sheet = ctx.page.locator('section[class*="rounded-t-2xl"]').first(); const sheetBox = await sheet.boundingBox().catch(() => null); if (!sheetBox) return; // desktop layout: nothing to drag const handle = ctx.page .locator('section[class*="rounded-t-2xl"] [class*="touch-none"]') .first(); const handleBox = (await handle.boundingBox().catch(() => null)) ?? { x: sheetBox.x, y: sheetBox.y + 4, width: sheetBox.width, height: 20, }; const viewport = ctx.page.viewportSize() ?? { width: 540, height: 960 }; const startX = handleBox.x + handleBox.width / 2; const startY = handleBox.y + handleBox.height / 2; // The sheet resizes so its top tracks the pointer (the component // clamps to its own min/max). Aim the pointer where the handle would // sit when the sheet covers `toHeightFrac` of the viewport. const handleOffset = startY - sheetBox.y; const endY = viewport.height * (1 - step.toHeightFrac) + handleOffset; const approachMs = Math.min(260, Math.max(140, Math.round(step.durationMs * 0.25))); const dragMs = Math.max(180, step.durationMs - approachMs - 80); await smoothMove(ctx.page, ctx.cursor, { x: startX, y: startY }, { durationMs: approachMs }); await ctx.page.mouse.down(); await smoothMove( ctx.page, { x: startX, y: startY }, { x: startX, y: endY }, { durationMs: dragMs, realMouse: true } ); await ctx.page.mouse.up(); ctx.cursor = { x: startX, y: endY }; return; } case 'openFilterGroup': // Click is idempotent: if the group is already expanded, the click // would collapse it, which we don't want. Detect via aria-expanded // (Radix Accordion sets it on the trigger) and skip the click when // the group is already open. await ctx.page.evaluate((selector) => { const trigger = document.querySelector(selector); if (!trigger) return; if (trigger.getAttribute('aria-expanded') === 'true') return; trigger.click(); }, step.selector); return; case 'pressKey': await ctx.page.keyboard.press(step.key); return; } } async function resolveTarget( ctx: ScriptCtx, target: Target ): Promise<{ x: number; y: number }> { if (target.kind === 'point') return { x: target.x, y: target.y }; if (target.kind === 'hexagon') { const targets = await ctx.dashboard.visibleHexagonTargets(1); if (targets.length === 0) throw new Error('No visible hexagon to target'); return { x: targets[0].x, y: targets[0].y }; } if (target.kind === 'viewportFraction') { const viewport = ctx.page.viewportSize() ?? { width: 1920, height: 1080 }; return { x: Math.round(Math.min(1, Math.max(0, target.xFrac)) * viewport.width), y: Math.round(Math.min(1, Math.max(0, target.yFrac)) * viewport.height), }; } // Bounded wait: a storyboard pointing at a missing element should fail in // seconds with a clear error, not stall for Playwright's default 30s. const box = await ctx.page.locator(target.selector).boundingBox({ timeout: 5000 }); if (!box) throw new Error(`No bounding box for selector: ${target.selector}`); return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; } async function tryResolveTarget( ctx: ScriptCtx, target: Target, timeoutMs: number ): Promise<{ x: number; y: number } | null> { if (target.kind !== 'element') { try { return await resolveTarget(ctx, target); } catch { return null; } } const locator = ctx.page.locator(target.selector).first(); try { await locator.waitFor({ state: 'visible', timeout: timeoutMs }); // Playwright's "visible" only means the element has a box; a filter-card // button can sit scrolled out of the bottom sheet's viewport, and a blind // click at its coordinates would land on the map behind the sheet. // Scrolling it into view first makes the click (and the recording of it) // honest. await locator.scrollIntoViewIfNeeded({ timeout: timeoutMs }).catch(() => {}); const box = await locator.boundingBox({ timeout: timeoutMs }); if (!box) return null; return { x: box.x + box.width / 2, y: box.y + box.height / 2 }; } catch { return null; } } /** * Load synth's measured cue durations. Falls back to a worst-case estimate * if the manifest is missing: that path is only used for ``--no-audio`` * runs, where the visual flow needs to play even without speech to time * against. */ function loadSynthIndex(storyboard: Storyboard): SynthCue[] { const path = join(OUTPUT_DIR, storyboard.name, 'audio', 'index.json'); if (existsSync(path)) { const raw = JSON.parse(readFileSync(path, 'utf-8')) as { items: SynthCue[]; }; const byIndex = new Map(raw.items.map((it) => [it.cueIndex, it] as const)); return storyboard.cues.map((cue, i) => { const m = byIndex.get(i); if (!m) { throw new Error( `Synth manifest is missing cue ${i} ("${cue.text.slice(0, 40)}…"). ` + `Re-run preflight + synth so the audio matches the storyboard.` ); } return m; }); } console.log( `[runner] no ${path} found. Using worst-case fallback durations (${FALLBACK_MS_PER_WORD}ms/word + ${FALLBACK_TAIL_BUFFER_MS}ms buffer). Audio will be missing.` ); return storyboard.cues.map((cue, cueIndex) => ({ cueIndex, text: cue.text, durationMs: estimateFallbackDurationMs(cue.text), })); } function estimateFallbackDurationMs(text: string): number { const wordCount = text.split(/\s+/).filter(Boolean).length; const cjkCount = text.match(/\p{Script=Han}/gu)?.length ?? 0; const units = Math.max(wordCount, Math.ceil(cjkCount / CJK_CHARS_PER_FALLBACK_WORD), 1); return units * FALLBACK_MS_PER_WORD + FALLBACK_TAIL_BUFFER_MS; } export type { Page };