541 lines
19 KiB
TypeScript
541 lines
19 KiB
TypeScript
import { existsSync, readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import type { Page } from 'playwright';
|
|
import { LEAD_IN_S, OUTPUT_DIR } from './config.js';
|
|
import {
|
|
clearVignette,
|
|
hideAdScene,
|
|
hideCaption,
|
|
scrollPaneTo,
|
|
setCursorScale,
|
|
showAdScene,
|
|
showCaption,
|
|
showOutro,
|
|
zoomReset,
|
|
zoomTo,
|
|
} from './dom.js';
|
|
import { fakeType, sleep, smoothDragSliderThumb, smoothMove } from './motion.js';
|
|
import { narrationLog } from './narration.js';
|
|
import type { Activity, Cue, ScriptCtx, Storyboard, Target } from './script.js';
|
|
|
|
export interface RunnerResult {
|
|
/** Wall-clock when the first activity started. */
|
|
sceneStartMs: number;
|
|
/** Wall-clock when the last activity finished (after padding). */
|
|
sceneEndMs: number;
|
|
}
|
|
|
|
const MAP_ZOOM_WHEEL_DELTA = -120;
|
|
const FALLBACK_MS_PER_WORD = 750;
|
|
const FALLBACK_TAIL_BUFFER_MS = 800;
|
|
const CJK_CHARS_PER_FALLBACK_WORD = 2;
|
|
|
|
interface SynthCue {
|
|
cueIndex: number;
|
|
text: string;
|
|
durationMs: number;
|
|
}
|
|
|
|
/**
|
|
* Drive the recording from a cue-anchored storyboard.
|
|
*
|
|
* Synth runs first and writes ``output/audio/index.json`` with per-cue
|
|
* measured durations. The runner reads that manifest and sizes each cue's
|
|
* wall-clock to its measured audio length: ``during`` activities run
|
|
* sequentially with their declared budgets, then a final wait pads to the
|
|
* full cue duration so the caption stays on for as long as the audio
|
|
* plays. ``tail`` activities run after the caption hides; ``gapBeforeMs``
|
|
* inserts pure silence before the next cue.
|
|
*
|
|
* The activity cursor is wall-clock honest: each step advances it by
|
|
* ``max(declared, actual)`` so an overrun extends the timeline rather than
|
|
* silently desyncing the narration manifest from reality. videoTimeMs
|
|
* recorded for each cue therefore matches the trimmed mp4 frame-for-frame,
|
|
* which is what the mux step needs to drop audio at the right moment.
|
|
*
|
|
* If the audio manifest is missing (``--no-audio`` runs), we fall back to a
|
|
* worst-case estimate (750ms/word + 800ms buffer) so the visual flow still
|
|
* works, just without sound.
|
|
*/
|
|
export async function runStoryboard(
|
|
ctx: ScriptCtx,
|
|
storyboard: Storyboard
|
|
): Promise<RunnerResult> {
|
|
narrationLog.reset();
|
|
|
|
const synth = loadSynthIndex(storyboard);
|
|
const sceneStartMs = Date.now();
|
|
const leadInMs = LEAD_IN_S * 1000;
|
|
const cursor = { ms: 0 };
|
|
|
|
for (const step of storyboard.pre ?? []) {
|
|
cursor.ms += await runStep(ctx, step);
|
|
}
|
|
|
|
for (let i = 0; i < storyboard.cues.length; i++) {
|
|
await runCue(ctx, storyboard.cues[i], synth[i], cursor, leadInMs);
|
|
}
|
|
|
|
for (const step of storyboard.post ?? []) {
|
|
cursor.ms += await runStep(ctx, step);
|
|
}
|
|
|
|
return { sceneStartMs, sceneEndMs: sceneStartMs + cursor.ms };
|
|
}
|
|
|
|
async function runCue(
|
|
ctx: ScriptCtx,
|
|
cue: Cue,
|
|
synth: SynthCue,
|
|
cursor: { ms: number },
|
|
leadInMs: number
|
|
): Promise<void> {
|
|
if (cue.gapBeforeMs > 0) {
|
|
await sleep(cue.gapBeforeMs);
|
|
cursor.ms += cue.gapBeforeMs;
|
|
}
|
|
|
|
const measuredAudioMs = synth.durationMs;
|
|
narrationLog.add({
|
|
text: cue.text,
|
|
videoTimeMs: cursor.ms + leadInMs,
|
|
durationMs: measuredAudioMs,
|
|
});
|
|
// The spoken line is never rendered; only an explicit short caption is.
|
|
if (cue.caption) {
|
|
await showCaption(ctx.page, cue.caption, cue.captionPlacement);
|
|
}
|
|
|
|
const during = cue.during ?? [];
|
|
const declaredSum = during.reduce((s, a) => s + a.durationMs, 0);
|
|
if (declaredSum > measuredAudioMs + 50) {
|
|
throw new Error(
|
|
`Cue ${synth.cueIndex} "${cue.text.slice(0, 40)}…" has ${declaredSum}ms of ` +
|
|
`during activities but the measured audio is only ${measuredAudioMs}ms. ` +
|
|
`Trim a during step, lengthen the cue text, or move work into tail.`
|
|
);
|
|
}
|
|
// Time the during block as a whole: individual steps may overrun their
|
|
// budgets, but what matters at the cue boundary is total wall-clock.
|
|
const duringStart = Date.now();
|
|
for (const step of during) {
|
|
await runStep(ctx, step);
|
|
}
|
|
const duringElapsed = Date.now() - duringStart;
|
|
if (duringElapsed < measuredAudioMs) {
|
|
await sleep(measuredAudioMs - duringElapsed);
|
|
cursor.ms += measuredAudioMs;
|
|
} else {
|
|
cursor.ms += duringElapsed;
|
|
}
|
|
|
|
if (cue.caption) {
|
|
await hideCaption(ctx.page);
|
|
}
|
|
|
|
for (const step of cue.tail ?? []) {
|
|
cursor.ms += await runStep(ctx, step);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Run a single activity. Pads short steps to their declared budget, lets
|
|
* long ones bleed past it, and returns ``max(declared, actual)`` so the
|
|
* caller can advance the wall-clock-honest cursor.
|
|
*/
|
|
async function runStep(ctx: ScriptCtx, step: Activity): Promise<number> {
|
|
const startedAt = Date.now();
|
|
await runActivity(ctx, step);
|
|
const realMs = Date.now() - startedAt;
|
|
if (realMs < step.durationMs) {
|
|
await sleep(step.durationMs - realMs);
|
|
return step.durationMs;
|
|
}
|
|
if (realMs > step.durationMs + 50) {
|
|
console.log(
|
|
`[runner] step ${step.kind} ran ${realMs}ms over a ${step.durationMs}ms budget (drift +${realMs - step.durationMs}ms)`
|
|
);
|
|
}
|
|
return realMs;
|
|
}
|
|
|
|
async function runActivity(ctx: ScriptCtx, step: Activity): Promise<void> {
|
|
switch (step.kind) {
|
|
case 'wait':
|
|
return;
|
|
case 'clearVignette':
|
|
await clearVignette(ctx.page);
|
|
return;
|
|
case 'zoomTo': {
|
|
const focus = await resolveTarget(ctx, step.target);
|
|
await zoomTo(ctx.page, {
|
|
scale: step.scale,
|
|
focusX: focus.x,
|
|
focusY: focus.y,
|
|
durationMs: step.durationMs,
|
|
});
|
|
return;
|
|
}
|
|
case 'zoomReset':
|
|
await zoomReset(ctx.page, step.durationMs);
|
|
return;
|
|
case 'cursorScale':
|
|
await setCursorScale(ctx.page, step.scale, step.durationMs);
|
|
return;
|
|
case 'moveCursor': {
|
|
const to = await resolveTarget(ctx, step.target);
|
|
await smoothMove(ctx.page, ctx.cursor, to, { durationMs: step.durationMs });
|
|
ctx.cursor = to;
|
|
return;
|
|
}
|
|
case 'click': {
|
|
const selectionVersion = ctx.dashboard.getSelectionStatsVersion();
|
|
// Hexagon targets are projected from the latest map response; make
|
|
// sure that response corresponds to the settled viewport (a zoom in
|
|
// the previous cue may still have postcode fetches in flight).
|
|
if (step.target.kind === 'hexagon') {
|
|
await ctx.dashboard.waitForApiIdle(3000);
|
|
}
|
|
const candidates =
|
|
step.target.kind === 'hexagon' && step.waitForSelectionReady
|
|
? await ctx.dashboard.visibleHexagonTargets(4)
|
|
: [await resolveTarget(ctx, step.target)];
|
|
let lastError: unknown = null;
|
|
|
|
for (let i = 0; i < candidates.length; i++) {
|
|
const to = candidates[i];
|
|
const moveMs = Math.max(120, Math.round(step.durationMs * 0.7));
|
|
await smoothMove(ctx.page, ctx.cursor, to, { durationMs: moveMs });
|
|
ctx.cursor = to;
|
|
await ctx.page.mouse.click(to.x, to.y);
|
|
if (!step.waitForSelectionReady) return;
|
|
|
|
try {
|
|
await ctx.dashboard.waitForSelectionReady(
|
|
selectionVersion,
|
|
Math.min(step.timeoutMs ?? 12000, i === candidates.length - 1 ? 12000 : 4000)
|
|
);
|
|
return;
|
|
} catch (err) {
|
|
lastError = err;
|
|
}
|
|
}
|
|
throw lastError ?? new Error('Click did not open the selection pane');
|
|
}
|
|
case 'clickIfVisible': {
|
|
const to = await tryResolveTarget(ctx, step.target, step.timeoutMs ?? 700);
|
|
if (!to) return;
|
|
const moveMs = Math.max(120, Math.round(step.durationMs * 0.7));
|
|
await smoothMove(ctx.page, ctx.cursor, to, { durationMs: moveMs });
|
|
ctx.cursor = to;
|
|
await ctx.page.mouse.click(to.x, to.y);
|
|
return;
|
|
}
|
|
case 'type':
|
|
await fakeType(ctx.page, step.selector, step.text, step.durationMs);
|
|
return;
|
|
case 'mapZoom': {
|
|
const point = await resolveTarget(ctx, step.target);
|
|
const mapVersion = ctx.dashboard.getMapDataVersion();
|
|
const delta = step.direction === 'out' ? -MAP_ZOOM_WHEEL_DELTA : MAP_ZOOM_WHEEL_DELTA;
|
|
const handled = await ctx.page.evaluate(
|
|
async ({ x, y, steps, durationMs, direction, center }) => {
|
|
const root = document.querySelector('.maplibregl-map') as HTMLElement | null;
|
|
const fiberKey = root
|
|
? Object.getOwnPropertyNames(root).find((key) => key.startsWith('__reactFiber$'))
|
|
: undefined;
|
|
let fiber = fiberKey ? (root as unknown as Record<string, unknown>)[fiberKey] : null;
|
|
let mapRef: unknown = null;
|
|
while (fiber && typeof fiber === 'object') {
|
|
const maybeFiber = fiber as {
|
|
ref?: { current?: unknown };
|
|
return?: unknown;
|
|
};
|
|
const current = maybeFiber.ref?.current;
|
|
if (
|
|
current &&
|
|
typeof current === 'object' &&
|
|
typeof (current as { getMap?: unknown }).getMap === 'function'
|
|
) {
|
|
mapRef = current;
|
|
break;
|
|
}
|
|
fiber = maybeFiber.return ?? null;
|
|
}
|
|
|
|
const map = (mapRef as { getMap?: () => unknown } | null)?.getMap?.();
|
|
if (!map || typeof map !== 'object') return false;
|
|
const mapApi = map as {
|
|
getCanvas: () => HTMLCanvasElement;
|
|
getZoom: () => number;
|
|
getMinZoom?: () => number;
|
|
getMaxZoom?: () => number;
|
|
unproject: (point: [number, number]) => unknown;
|
|
zoomTo: (
|
|
zoom: number,
|
|
options: { around?: unknown; duration?: number; essential?: boolean }
|
|
) => void;
|
|
easeTo?: (options: {
|
|
center?: unknown;
|
|
zoom?: number;
|
|
duration?: number;
|
|
essential?: boolean;
|
|
}) => void;
|
|
};
|
|
if (
|
|
typeof mapApi.getCanvas !== 'function' ||
|
|
typeof mapApi.getZoom !== 'function' ||
|
|
typeof mapApi.unproject !== 'function' ||
|
|
typeof mapApi.zoomTo !== 'function'
|
|
) {
|
|
return false;
|
|
}
|
|
|
|
const rect = mapApi.getCanvas().getBoundingClientRect();
|
|
const around = mapApi.unproject([x - rect.left, y - rect.top]);
|
|
const sign = direction === 'out' ? -1 : 1;
|
|
const zoomDelta = Math.max(0.25, Math.min(5.2, steps * 0.28)) * sign;
|
|
const minZoom = mapApi.getMinZoom?.() ?? 0;
|
|
const maxZoom = mapApi.getMaxZoom?.() ?? 22;
|
|
const targetZoom = Math.max(minZoom, Math.min(maxZoom, mapApi.getZoom() + zoomDelta));
|
|
if (center && typeof mapApi.easeTo === 'function') {
|
|
mapApi.easeTo({
|
|
center: around,
|
|
zoom: targetZoom,
|
|
duration: durationMs,
|
|
essential: true,
|
|
});
|
|
} else {
|
|
mapApi.zoomTo(targetZoom, { around, duration: durationMs, essential: true });
|
|
}
|
|
await new Promise((resolve) => window.setTimeout(resolve, durationMs));
|
|
return true;
|
|
},
|
|
{
|
|
x: point.x,
|
|
y: point.y,
|
|
steps: step.steps,
|
|
durationMs: step.durationMs,
|
|
direction: step.direction,
|
|
center: step.center ?? false,
|
|
}
|
|
);
|
|
|
|
if (!handled) {
|
|
const perStepMs = Math.floor(step.durationMs / Math.max(1, step.steps));
|
|
await ctx.page.evaluate(
|
|
async ({ x, y, steps, durationMs, delta }) => {
|
|
const wait = (ms: number) =>
|
|
new Promise<void>((resolve) => window.setTimeout(resolve, ms));
|
|
const perStep = Math.floor(durationMs / Math.max(1, steps));
|
|
for (let i = 0; i < steps; i++) {
|
|
const target = document.elementFromPoint(x, y) ?? document.querySelector('canvas');
|
|
target?.dispatchEvent(
|
|
new WheelEvent('wheel', {
|
|
bubbles: true,
|
|
cancelable: true,
|
|
clientX: x,
|
|
clientY: y,
|
|
deltaY: delta,
|
|
deltaMode: WheelEvent.DOM_DELTA_PIXEL,
|
|
view: window,
|
|
})
|
|
);
|
|
if (perStep > 0) await wait(perStep);
|
|
}
|
|
},
|
|
{ x: point.x, y: point.y, steps: step.steps, durationMs: step.durationMs, delta }
|
|
);
|
|
if (perStepMs > 0) await sleep(0);
|
|
}
|
|
if (step.waitForMapSettled) {
|
|
await ctx.dashboard.waitForMapSettled(mapVersion, step.timeoutMs ?? 12000);
|
|
}
|
|
return;
|
|
}
|
|
case 'dragSlider': {
|
|
const mapVersion = ctx.dashboard.getMapDataVersion();
|
|
ctx.cursor = await smoothDragSliderThumb(
|
|
ctx.page,
|
|
step.thumbSelector,
|
|
step.trackSelector,
|
|
ctx.cursor,
|
|
step.toFraction,
|
|
step.durationMs
|
|
);
|
|
if (step.waitForMapSettled) {
|
|
await ctx.dashboard.waitForMapSettled(mapVersion, step.timeoutMs ?? 12000);
|
|
}
|
|
return;
|
|
}
|
|
case 'submitForm': {
|
|
const mapVersion = ctx.dashboard.getMapDataVersion();
|
|
await ctx.page.evaluate((selector) => {
|
|
document.querySelector<HTMLFormElement>(selector)?.requestSubmit();
|
|
}, step.formSelector);
|
|
if (step.waitForMapSettled) {
|
|
await ctx.dashboard.waitForMapSettled(mapVersion, step.timeoutMs ?? 12000);
|
|
}
|
|
return;
|
|
}
|
|
case 'showOutro':
|
|
await showOutro(ctx.page, step.brand, step.tagline, step.url, step.subtitle);
|
|
return;
|
|
case 'showAdScene':
|
|
await showAdScene(ctx.page, step.scene);
|
|
return;
|
|
case 'hideAdScene':
|
|
await hideAdScene(ctx.page);
|
|
return;
|
|
case 'scrollPane':
|
|
await scrollPaneTo(ctx.page, step.selector, step.top);
|
|
return;
|
|
case 'dragSheet': {
|
|
const sheet = ctx.page.locator('section[class*="rounded-t-2xl"]').first();
|
|
const sheetBox = await sheet.boundingBox().catch(() => null);
|
|
if (!sheetBox) return; // desktop layout: nothing to drag
|
|
const handle = ctx.page
|
|
.locator('section[class*="rounded-t-2xl"] [class*="touch-none"]')
|
|
.first();
|
|
const handleBox = (await handle.boundingBox().catch(() => null)) ?? {
|
|
x: sheetBox.x,
|
|
y: sheetBox.y + 4,
|
|
width: sheetBox.width,
|
|
height: 20,
|
|
};
|
|
const viewport = ctx.page.viewportSize() ?? { width: 540, height: 960 };
|
|
const startX = handleBox.x + handleBox.width / 2;
|
|
const startY = handleBox.y + handleBox.height / 2;
|
|
// The sheet resizes so its top tracks the pointer (the component
|
|
// clamps to its own min/max). Aim the pointer where the handle would
|
|
// sit when the sheet covers `toHeightFrac` of the viewport.
|
|
const handleOffset = startY - sheetBox.y;
|
|
const endY = viewport.height * (1 - step.toHeightFrac) + handleOffset;
|
|
const approachMs = Math.min(260, Math.max(140, Math.round(step.durationMs * 0.25)));
|
|
const dragMs = Math.max(180, step.durationMs - approachMs - 80);
|
|
await smoothMove(ctx.page, ctx.cursor, { x: startX, y: startY }, { durationMs: approachMs });
|
|
await ctx.page.mouse.down();
|
|
await smoothMove(
|
|
ctx.page,
|
|
{ x: startX, y: startY },
|
|
{ x: startX, y: endY },
|
|
{ durationMs: dragMs, realMouse: true }
|
|
);
|
|
await ctx.page.mouse.up();
|
|
ctx.cursor = { x: startX, y: endY };
|
|
return;
|
|
}
|
|
case 'openFilterGroup':
|
|
// Click is idempotent: if the group is already expanded, the click
|
|
// would collapse it, which we don't want. Detect via aria-expanded
|
|
// (Radix Accordion sets it on the trigger) and skip the click when
|
|
// the group is already open.
|
|
await ctx.page.evaluate((selector) => {
|
|
const trigger = document.querySelector<HTMLElement>(selector);
|
|
if (!trigger) return;
|
|
if (trigger.getAttribute('aria-expanded') === 'true') return;
|
|
trigger.click();
|
|
}, step.selector);
|
|
return;
|
|
case 'pressKey':
|
|
await ctx.page.keyboard.press(step.key);
|
|
return;
|
|
}
|
|
}
|
|
|
|
async function resolveTarget(
|
|
ctx: ScriptCtx,
|
|
target: Target
|
|
): Promise<{ x: number; y: number }> {
|
|
if (target.kind === 'point') return { x: target.x, y: target.y };
|
|
if (target.kind === 'hexagon') {
|
|
const targets = await ctx.dashboard.visibleHexagonTargets(1);
|
|
if (targets.length === 0) throw new Error('No visible hexagon to target');
|
|
return { x: targets[0].x, y: targets[0].y };
|
|
}
|
|
if (target.kind === 'viewportFraction') {
|
|
const viewport = ctx.page.viewportSize() ?? { width: 1920, height: 1080 };
|
|
return {
|
|
x: Math.round(Math.min(1, Math.max(0, target.xFrac)) * viewport.width),
|
|
y: Math.round(Math.min(1, Math.max(0, target.yFrac)) * viewport.height),
|
|
};
|
|
}
|
|
// Bounded wait: a storyboard pointing at a missing element should fail in
|
|
// seconds with a clear error, not stall for Playwright's default 30s.
|
|
const box = await ctx.page.locator(target.selector).boundingBox({ timeout: 5000 });
|
|
if (!box) throw new Error(`No bounding box for selector: ${target.selector}`);
|
|
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
}
|
|
|
|
async function tryResolveTarget(
|
|
ctx: ScriptCtx,
|
|
target: Target,
|
|
timeoutMs: number
|
|
): Promise<{ x: number; y: number } | null> {
|
|
if (target.kind !== 'element') {
|
|
try {
|
|
return await resolveTarget(ctx, target);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
const locator = ctx.page.locator(target.selector).first();
|
|
try {
|
|
await locator.waitFor({ state: 'visible', timeout: timeoutMs });
|
|
// Playwright's "visible" only means the element has a box; a filter-card
|
|
// button can sit scrolled out of the bottom sheet's viewport, and a blind
|
|
// click at its coordinates would land on the map behind the sheet.
|
|
// Scrolling it into view first makes the click (and the recording of it)
|
|
// honest.
|
|
await locator.scrollIntoViewIfNeeded({ timeout: timeoutMs }).catch(() => {});
|
|
const box = await locator.boundingBox({ timeout: timeoutMs });
|
|
if (!box) return null;
|
|
return { x: box.x + box.width / 2, y: box.y + box.height / 2 };
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Load synth's measured cue durations. Falls back to a worst-case estimate
|
|
* if the manifest is missing: that path is only used for ``--no-audio``
|
|
* runs, where the visual flow needs to play even without speech to time
|
|
* against.
|
|
*/
|
|
function loadSynthIndex(storyboard: Storyboard): SynthCue[] {
|
|
const path = join(OUTPUT_DIR, storyboard.name, 'audio', 'index.json');
|
|
if (existsSync(path)) {
|
|
const raw = JSON.parse(readFileSync(path, 'utf-8')) as {
|
|
items: SynthCue[];
|
|
};
|
|
const byIndex = new Map(raw.items.map((it) => [it.cueIndex, it] as const));
|
|
return storyboard.cues.map((cue, i) => {
|
|
const m = byIndex.get(i);
|
|
if (!m) {
|
|
throw new Error(
|
|
`Synth manifest is missing cue ${i} ("${cue.text.slice(0, 40)}…"). ` +
|
|
`Re-run preflight + synth so the audio matches the storyboard.`
|
|
);
|
|
}
|
|
return m;
|
|
});
|
|
}
|
|
console.log(
|
|
`[runner] no ${path} found. Using worst-case fallback durations (${FALLBACK_MS_PER_WORD}ms/word + ${FALLBACK_TAIL_BUFFER_MS}ms buffer). Audio will be missing.`
|
|
);
|
|
return storyboard.cues.map((cue, cueIndex) => ({
|
|
cueIndex,
|
|
text: cue.text,
|
|
durationMs: estimateFallbackDurationMs(cue.text),
|
|
}));
|
|
}
|
|
|
|
function estimateFallbackDurationMs(text: string): number {
|
|
const wordCount = text.split(/\s+/).filter(Boolean).length;
|
|
const cjkCount = text.match(/\p{Script=Han}/gu)?.length ?? 0;
|
|
const units = Math.max(wordCount, Math.ceil(cjkCount / CJK_CHARS_PER_FALLBACK_WORD), 1);
|
|
return units * FALLBACK_MS_PER_WORD + FALLBACK_TAIL_BUFFER_MS;
|
|
}
|
|
|
|
export type { Page };
|