More FE changes

video/src/runner.ts

@@ -0,0 +1,275 @@
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import type { Page } from 'playwright';
+import { LEAD_IN_S, OUTPUT_DIR } from './config.js';
+import {
+  clearVignette,
+  hideCaption,
+  setCursorScale,
+  showCaption,
+  showOutro,
+  zoomReset,
+  zoomTo,
+} from './dom.js';
+import { fakeType, sleep, smoothDragSliderThumb, smoothMove } from './motion.js';
+import { narrationLog } from './narration.js';
+import type { Activity, Cue, ScriptCtx, Storyboard, Target } from './script.js';
+
+export interface RunnerResult {
+  /** Wall-clock when the first activity started. */
+  sceneStartMs: number;
+  /** Wall-clock when the last activity finished (after padding). */
+  sceneEndMs: number;
+}
+
+const MAP_ZOOM_WHEEL_DELTA = -120;
+const FALLBACK_MS_PER_WORD = 750;
+const FALLBACK_TAIL_BUFFER_MS = 800;
+
+interface SynthCue {
+  cueIndex: number;
+  text: string;
+  durationMs: number;
+}
+
+/**
+ * Drive the recording from a cue-anchored storyboard.
+ *
+ * Synth runs first and writes ``output/audio/index.json`` with per-cue
+ * measured durations. The runner reads that manifest and sizes each cue's
+ * wall-clock to its measured audio length: ``during`` activities run
+ * sequentially with their declared budgets, then a final wait pads to the
+ * full cue duration so the caption stays on for as long as the audio
+ * plays. ``tail`` activities run after the caption hides; ``gapBeforeMs``
+ * inserts pure silence before the next cue.
+ *
+ * The activity cursor is wall-clock honest: each step advances it by
+ * ``max(declared, actual)`` so an overrun extends the timeline rather than
+ * silently desyncing the narration manifest from reality. videoTimeMs
+ * recorded for each cue therefore matches the trimmed mp4 frame-for-frame,
+ * which is what the mux step needs to drop audio at the right moment.
+ *
+ * If the audio manifest is missing (``--no-audio`` runs), we fall back to a
+ * worst-case estimate (750ms/word + 800ms buffer) so the visual flow still
+ * works, just without sound.
+ */
+export async function runStoryboard(
+  ctx: ScriptCtx,
+  storyboard: Storyboard
+): Promise<RunnerResult> {
+  narrationLog.reset();
+
+  const synth = loadSynthIndex(storyboard);
+  const sceneStartMs = Date.now();
+  const leadInMs = LEAD_IN_S * 1000;
+  const cursor = { ms: 0 };
+
+  for (const step of storyboard.pre ?? []) {
+    cursor.ms += await runStep(ctx, step);
+  }
+
+  for (let i = 0; i < storyboard.cues.length; i++) {
+    await runCue(ctx, storyboard.cues[i], synth[i], cursor, leadInMs);
+  }
+
+  for (const step of storyboard.post ?? []) {
+    cursor.ms += await runStep(ctx, step);
+  }
+
+  return { sceneStartMs, sceneEndMs: sceneStartMs + cursor.ms };
+}
+
+async function runCue(
+  ctx: ScriptCtx,
+  cue: Cue,
+  synth: SynthCue,
+  cursor: { ms: number },
+  leadInMs: number
+): Promise<void> {
+  if (cue.gapBeforeMs > 0) {
+    await sleep(cue.gapBeforeMs);
+    cursor.ms += cue.gapBeforeMs;
+  }
+
+  const measuredAudioMs = synth.durationMs;
+  narrationLog.add({
+    text: cue.text,
+    videoTimeMs: cursor.ms + leadInMs,
+    durationMs: measuredAudioMs,
+  });
+  await showCaption(ctx.page, cue.text);
+
+  const during = cue.during ?? [];
+  const declaredSum = during.reduce((s, a) => s + a.durationMs, 0);
+  if (declaredSum > measuredAudioMs + 50) {
+    throw new Error(
+      `Cue ${synth.cueIndex} "${cue.text.slice(0, 40)}…" has ${declaredSum}ms of ` +
+        `during activities but the measured audio is only ${measuredAudioMs}ms. ` +
+        `Trim a during step, lengthen the cue text, or move work into tail.`
+    );
+  }
+  // Time the during block as a whole — individual steps may overrun their
+  // budgets, but what matters at the cue boundary is total wall-clock.
+  const duringStart = Date.now();
+  for (const step of during) {
+    await runStep(ctx, step);
+  }
+  const duringElapsed = Date.now() - duringStart;
+  if (duringElapsed < measuredAudioMs) {
+    await sleep(measuredAudioMs - duringElapsed);
+    cursor.ms += measuredAudioMs;
+  } else {
+    cursor.ms += duringElapsed;
+  }
+
+  await hideCaption(ctx.page);
+
+  for (const step of cue.tail ?? []) {
+    cursor.ms += await runStep(ctx, step);
+  }
+}
+
+/**
+ * Run a single activity. Pads short steps to their declared budget, lets
+ * long ones bleed past it, and returns ``max(declared, actual)`` so the
+ * caller can advance the wall-clock-honest cursor.
+ */
+async function runStep(ctx: ScriptCtx, step: Activity): Promise<number> {
+  const startedAt = Date.now();
+  await runActivity(ctx, step);
+  const realMs = Date.now() - startedAt;
+  if (realMs < step.durationMs) {
+    await sleep(step.durationMs - realMs);
+    return step.durationMs;
+  }
+  if (realMs > step.durationMs + 50) {
+    console.log(
+      `[runner] step ${step.kind} ran ${realMs}ms over a ${step.durationMs}ms budget (drift +${realMs - step.durationMs}ms)`
+    );
+  }
+  return realMs;
+}
+
+async function runActivity(ctx: ScriptCtx, step: Activity): Promise<void> {
+  switch (step.kind) {
+    case 'wait':
+      return;
+    case 'clearVignette':
+      await clearVignette(ctx.page);
+      return;
+    case 'zoomTo': {
+      const focus = await resolveTarget(ctx, step.target);
+      await zoomTo(ctx.page, {
+        scale: step.scale,
+        focusX: focus.x,
+        focusY: focus.y,
+        durationMs: step.durationMs,
+      });
+      return;
+    }
+    case 'zoomReset':
+      await zoomReset(ctx.page, step.durationMs);
+      return;
+    case 'cursorScale':
+      await setCursorScale(ctx.page, step.scale, step.durationMs);
+      return;
+    case 'moveCursor': {
+      const to = await resolveTarget(ctx, step.target);
+      await smoothMove(ctx.page, ctx.cursor, to, { durationMs: step.durationMs });
+      ctx.cursor = to;
+      return;
+    }
+    case 'click': {
+      const to = await resolveTarget(ctx, step.target);
+      const moveMs = Math.max(120, Math.round(step.durationMs * 0.7));
+      await smoothMove(ctx.page, ctx.cursor, to, { durationMs: moveMs });
+      ctx.cursor = to;
+      await ctx.page.mouse.click(to.x, to.y);
+      return;
+    }
+    case 'type':
+      await fakeType(ctx.page, step.selector, step.text, step.durationMs);
+      return;
+    case 'mapZoom': {
+      const point = await resolveTarget(ctx, step.target);
+      await ctx.page.mouse.move(point.x, point.y);
+      const perStepMs = Math.floor(step.durationMs / Math.max(1, step.steps));
+      for (let i = 0; i < step.steps; i++) {
+        await ctx.page.mouse.wheel(0, MAP_ZOOM_WHEEL_DELTA);
+        if (perStepMs > 0) await sleep(perStepMs);
+      }
+      return;
+    }
+    case 'dragSlider':
+      ctx.cursor = await smoothDragSliderThumb(
+        ctx.page,
+        step.thumbSelector,
+        step.trackSelector,
+        ctx.cursor,
+        step.toFraction,
+        step.durationMs
+      );
+      return;
+    case 'submitForm':
+      await ctx.page.evaluate((selector) => {
+        document.querySelector<HTMLFormElement>(selector)?.requestSubmit();
+      }, step.formSelector);
+      return;
+    case 'showOutro':
+      await showOutro(ctx.page, step.brand, step.tagline, step.url);
+      return;
+  }
+}
+
+async function resolveTarget(
+  ctx: ScriptCtx,
+  target: Target
+): Promise<{ x: number; y: number }> {
+  if (target.kind === 'point') return { x: target.x, y: target.y };
+  if (target.kind === 'hexagon') {
+    const targets = await ctx.dashboard.visibleHexagonTargets(1);
+    if (targets.length === 0) throw new Error('No visible hexagon to target');
+    return { x: targets[0].x, y: targets[0].y };
+  }
+  const box = await ctx.page.locator(target.selector).boundingBox();
+  if (!box) throw new Error(`No bounding box for selector: ${target.selector}`);
+  return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
+}
+
+/**
+ * Load synth's measured cue durations. Falls back to a worst-case estimate
+ * if the manifest is missing — that path is only used for ``--no-audio``
+ * runs, where the visual flow needs to play even without speech to time
+ * against.
+ */
+function loadSynthIndex(storyboard: Storyboard): SynthCue[] {
+  const path = join(OUTPUT_DIR, 'audio', 'index.json');
+  if (existsSync(path)) {
+    const raw = JSON.parse(readFileSync(path, 'utf-8')) as {
+      items: SynthCue[];
+    };
+    const byIndex = new Map(raw.items.map((it) => [it.cueIndex, it] as const));
+    return storyboard.cues.map((cue, i) => {
+      const m = byIndex.get(i);
+      if (!m) {
+        throw new Error(
+          `Synth manifest is missing cue ${i} ("${cue.text.slice(0, 40)}…"). ` +
+            `Re-run preflight + synth so the audio matches the storyboard.`
+        );
+      }
+      return m;
+    });
+  }
+  console.log(
+    `[runner] no ${path} found — using worst-case fallback durations (${FALLBACK_MS_PER_WORD}ms/word + ${FALLBACK_TAIL_BUFFER_MS}ms buffer). Audio will be missing.`
+  );
+  return storyboard.cues.map((cue, cueIndex) => ({
+    cueIndex,
+    text: cue.text,
+    durationMs:
+      cue.text.split(/\s+/).filter(Boolean).length * FALLBACK_MS_PER_WORD +
+      FALLBACK_TAIL_BUFFER_MS,
+  }));
+}
+
+export type { Page };