diff --git a/video/render.sh b/video/render.sh index df85c8d..fdff5b5 100755 --- a/video/render.sh +++ b/video/render.sh @@ -372,6 +372,16 @@ if [ "$DO_AUDIO" = "1" ]; then done fi +# -- cue timing gate --------------------------------------------------------- +# Every cue's `during` block has to fit inside the narration line it plays +# under, and synth has just measured every one of those lines. Check the whole +# set here, while the run has cost nothing but TTS. The recorder enforces the +# same rule, but it hits it one cue at a time, mid-take, after re-recording +# every storyboard that came before: a set of mistuned cues costs one full +# render cycle each to discover. This reports all of them at once. +say "Checking cue timings against the measured narration" +node dist/check-timing.js || fail "cue timing check failed (see the list above)" + # -- record ------------------------------------------------------------------ # record.ts iterates over storyboards in-process and writes per-storyboard # recording.webm + narration.json. One Node invocation handles all of them diff --git a/video/src/auth.ts b/video/src/auth.ts index 822f44c..85129cc 100644 --- a/video/src/auth.ts +++ b/video/src/auth.ts @@ -1,5 +1,5 @@ import { chromium } from 'playwright'; -import { APP_URL, AUTH_STATE_PATH } from './config.js'; +import { appUrl, AUTH_STATE_PATH } from './config.js'; import { ensureRecorderAdminUser } from './pb-admin.js'; /** @@ -27,7 +27,7 @@ async function programmatic() { const browser = await chromium.launch({ headless: true }); const context = await browser.newContext({ viewport: { width: 1280, height: 800 } }); const page = await context.newPage(); - await page.goto(APP_URL); + await page.goto(appUrl()); // Drive the dashboard's actual login modal. Scoping to
avoids the // modal's own "Log in" submit button (same text, different element). @@ -62,7 +62,7 @@ async function interactive() { const browser = await chromium.launch({ headless: false }); const context = await browser.newContext({ viewport: { width: 1280, height: 800 } }); const page = await context.newPage(); - await page.goto(APP_URL); + await page.goto(appUrl()); console.log(''); console.log(' → Log in via the UI in the opened browser window.'); diff --git a/video/src/check-timing.ts b/video/src/check-timing.ts new file mode 100644 index 0000000..fd2b66e --- /dev/null +++ b/video/src/check-timing.ts @@ -0,0 +1,89 @@ +import { + SLACK_WARN_MS, + cueTiming, + overrunMessage, + readSynthIndex, +} from './cue-timing.js'; +import { storyboards } from './storyboard.js'; + +/** + * Fail the render before recording if any cue's `during` block is longer than + * the narration line it plays under. + * + * Runs after synth, when every measured duration is known and nothing + * expensive has happened yet. Reports EVERY offending cue across EVERY + * storyboard in one pass: the point is to hand over the whole list, because + * the alternative (the runner's mid-take throw) reveals exactly one cue per + * render cycle and each cycle costs a full re-record of everything before it. + * + * Also warns about cues that fit only barely. Those are the ones that break + * next time someone touches the copy or the voice, and a warning here is + * cheaper than finding out from a failed run. + * + * Skips silently on --no-audio runs, where there are no measured durations to + * check against and the runner falls back to estimates. + */ +function main(): void { + const errors: string[] = []; + const warnings: string[] = []; + let checkedCues = 0; + let checkedStoryboards = 0; + + for (const storyboard of storyboards) { + // A storyboard whose manifest no longer matches its cues (a cue added + // without a re-synth) throws out of readSynthIndex. Collect that as a + // finding rather than letting it abort the sweep: one unreadable + // storyboard must not hide the other 22. + let synth; + try { + synth = readSynthIndex(storyboard); + } catch (err) { + errors.push(`[${storyboard.name}] ${err instanceof Error ? err.message : String(err)}`); + continue; + } + if (!synth) continue; + checkedStoryboards++; + + for (let i = 0; i < storyboard.cues.length; i++) { + const cue = storyboard.cues[i]; + const timing = cueTiming(cue, i, synth[i].durationMs); + checkedCues++; + if (timing.over) { + errors.push(`[${storyboard.name}] ${overrunMessage(cue, timing)}`); + } else if (timing.slackMs < SLACK_WARN_MS) { + warnings.push( + `[${storyboard.name}] cue ${i} fits with only ${timing.slackMs}ms to spare ` + + `(during=${timing.declaredMs}ms, audio=${timing.measuredMs}ms): ` + + `"${cue.text.slice(0, 40)}…"` + ); + } + } + } + + // Only a clean sweep that found nothing to read is a --no-audio run. Test + // errors first: every storyboard failing to load is the loudest possible + // signal, and reporting it as "nothing to check" would wave the render + // through on the strength of its own broken manifests. + if (checkedStoryboards === 0 && errors.length === 0) { + console.log('[check-timing] no synthesised audio found, skipping (--no-audio run)'); + return; + } + + for (const warning of warnings) console.warn(`[check-timing] WARN: ${warning}`); + + if (errors.length > 0) { + for (const error of errors) console.error(`[check-timing] FAIL: ${error}`); + console.error( + `\n[check-timing] ${errors.length} cue(s) declare more visual activity than their ` + + `narration covers. Fix them all now: recording would hit them one at a time.` + ); + process.exit(1); + } + + console.log( + `[check-timing] ${checkedCues} cues across ${checkedStoryboards} storyboards fit their ` + + `narration${warnings.length > 0 ? ` (${warnings.length} with under ${SLACK_WARN_MS}ms of slack)` : ''}` + ); +} + +main(); diff --git a/video/src/config.ts b/video/src/config.ts index 1ac4a2b..aaaa676 100644 --- a/video/src/config.ts +++ b/video/src/config.ts @@ -10,7 +10,14 @@ function requiredEnv(name: string): string { // voice, prompts, brand…) lives on the Storyboard object itself. See // src/storyboard.ts. -export const APP_URL = requiredEnv('APP_URL'); +/** + * Read lazily, not at import time. This module also holds pure constants + * (OUTPUT_DIR, LEAD_IN_S) that the offline steps need, and a top-level + * `requiredEnv` made importing any of them fail without a frontend URL: it + * put a live-stack dependency on dist/check-timing.js, which only reads JSON + * off disk. Only the steps that actually open a browser call this. + */ +export const appUrl = (): string => requiredEnv('APP_URL'); export const DASHBOARD_PATH = '/dashboard'; // Per-target storage state. render.sh sets AUTH_STATE_FILE to auth.local.json diff --git a/video/src/cue-timing.ts b/video/src/cue-timing.ts new file mode 100644 index 0000000..8581b87 --- /dev/null +++ b/video/src/cue-timing.ts @@ -0,0 +1,132 @@ +import { existsSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { OUTPUT_DIR } from './config.js'; +import type { Cue, Storyboard } from './script.js'; + +/** + * The cue/audio fit rule, shared by the runner (which enforces it per cue, + * mid-take) and dist/check-timing.js (which enforces it for every cue of + * every storyboard before a single frame is recorded). + * + * Both call sites used to be one call site: the runner threw the moment it + * reached a cue whose `during` block was longer than its measured audio. That + * is a true invariant but a terrible place to learn about it. Recording is the + * expensive stage, storyboards record one after another, and the throw kills + * the whole run, so a set of N mistuned cues cost N full re-runs to find, one + * per cycle, each one paying for the storyboards that already passed. The + * durations that decide the rule are all known straight after synth, so the + * check belongs there. The runner keeps its throw as a backstop for anyone + * running dist/record.js directly. + */ + +export interface SynthCue { + cueIndex: number; + text: string; + durationMs: number; +} + +const FALLBACK_MS_PER_WORD = 750; +const FALLBACK_TAIL_BUFFER_MS = 800; +const CJK_CHARS_PER_FALLBACK_WORD = 2; + +/** + * Slop allowed on the fit rule. A `during` block this much over its cue's + * audio is treated as authored-to-fit rather than a mistake: the runner pads + * each step up to its declared budget but never trims one, so a cue that + * overshoots by less than a frame or two of scheduler jitter would produce an + * identical cut either way. + */ +export const DURING_TOLERANCE_MS = 50; + +/** + * Cues with less slack than this fit today only because the TTS happened to + * speak the line slowly enough. They are not errors, but any re-synth (a + * reworded line, a new voice, a re-rolled seed) can push them over, so + * check-timing surfaces them while the render is still cheap to fix. + */ +export const SLACK_WARN_MS = 250; + +export interface CueTiming { + cueIndex: number; + /** Sum of the cue's declared `during` budgets. */ + declaredMs: number; + /** Measured length of the cue's synthesised audio. */ + measuredMs: number; + /** measured - declared. Negative means the visuals outlast the line. */ + slackMs: number; + /** True when the overrun is past DURING_TOLERANCE_MS, i.e. a hard error. */ + over: boolean; +} + +export function declaredDuringMs(cue: Cue): number { + return (cue.during ?? []).reduce((sum, activity) => sum + activity.durationMs, 0); +} + +export function cueTiming(cue: Cue, cueIndex: number, measuredMs: number): CueTiming { + const declaredMs = declaredDuringMs(cue); + return { + cueIndex, + declaredMs, + measuredMs, + slackMs: measuredMs - declaredMs, + over: declaredMs > measuredMs + DURING_TOLERANCE_MS, + }; +} + +/** The operator-facing explanation of an over-budget cue. */ +export function overrunMessage(cue: Cue, timing: CueTiming): string { + return ( + `Cue ${timing.cueIndex} "${cue.text.slice(0, 40)}…" has ${timing.declaredMs}ms of ` + + `during activities but the measured audio is only ${timing.measuredMs}ms ` + + `(${-timing.slackMs}ms over). Trim a during step, move staging into the ` + + `previous cue's tail, or lengthen the cue text.` + ); +} + +/** + * Read synth's measured cue durations for one storyboard, or null when the + * storyboard has no audio yet (a --no-audio run). + */ +export function readSynthIndex(storyboard: Storyboard): SynthCue[] | null { + const path = join(OUTPUT_DIR, storyboard.name, 'audio', 'index.json'); + if (!existsSync(path)) return null; + const raw = JSON.parse(readFileSync(path, 'utf-8')) as { items: SynthCue[] }; + const byIndex = new Map(raw.items.map((item) => [item.cueIndex, item] as const)); + return storyboard.cues.map((cue, i) => { + const measured = byIndex.get(i); + if (!measured) { + throw new Error( + `[${storyboard.name}] synth manifest is missing cue ${i} ("${cue.text.slice(0, 40)}…"). ` + + `Re-run preflight + synth so the audio matches the storyboard.` + ); + } + return measured; + }); +} + +/** + * Measured durations when synth has run, worst-case estimates when it has + * not. The estimate path only serves --no-audio runs, where the visual flow + * still has to play without speech to time against. + */ +export function loadSynthIndex(storyboard: Storyboard): SynthCue[] { + const measured = readSynthIndex(storyboard); + if (measured) return measured; + console.log( + `[runner] no ${join(OUTPUT_DIR, storyboard.name, 'audio', 'index.json')} found. ` + + `Using worst-case fallback durations (${FALLBACK_MS_PER_WORD}ms/word + ` + + `${FALLBACK_TAIL_BUFFER_MS}ms buffer). Audio will be missing.` + ); + return storyboard.cues.map((cue, cueIndex) => ({ + cueIndex, + text: cue.text, + durationMs: estimateFallbackDurationMs(cue.text), + })); +} + +function estimateFallbackDurationMs(text: string): number { + const wordCount = text.split(/\s+/).filter(Boolean).length; + const cjkCount = text.match(/\p{Script=Han}/gu)?.length ?? 0; + const units = Math.max(wordCount, Math.ceil(cjkCount / CJK_CHARS_PER_FALLBACK_WORD), 1); + return units * FALLBACK_MS_PER_WORD + FALLBACK_TAIL_BUFFER_MS; +} diff --git a/video/src/probe.ts b/video/src/probe.ts index 88d65f1..09f2812 100644 --- a/video/src/probe.ts +++ b/video/src/probe.ts @@ -1,5 +1,5 @@ import { chromium } from 'playwright'; -import { APP_URL, AUTH_STATE_PATH, DASHBOARD_PATH } from './config.js'; +import { appUrl, AUTH_STATE_PATH, DASHBOARD_PATH } from './config.js'; import { viewportFor } from './script.js'; import { storyboards } from './storyboard.js'; @@ -24,7 +24,7 @@ async function main() { } }); - await page.goto(`${APP_URL}${DASHBOARD_PATH}`, { waitUntil: 'networkidle' }); + await page.goto(`${appUrl()}${DASHBOARD_PATH}`, { waitUntil: 'networkidle' }); await new Promise((r) => setTimeout(r, 1500)); await page.screenshot({ path: 'output/probe-1-initial.png', fullPage: false }); diff --git a/video/src/routes.ts b/video/src/routes.ts index ad7ccd5..08dd57b 100644 --- a/video/src/routes.ts +++ b/video/src/routes.ts @@ -1,5 +1,5 @@ import type { Page } from 'playwright'; -import { APP_URL, DASHBOARD_PATH } from './config.js'; +import { appUrl, DASHBOARD_PATH } from './config.js'; import type { Storyboard } from './script.js'; export async function installDemoRoutes(page: Page, storyboard: Storyboard) { @@ -23,9 +23,14 @@ export function dashboardUrl(storyboard: Storyboard): string { params.append('filter', entry); } for (const tt of storyboard.content.stubbedTravelTimeFilters) { - params.append('tt', `${tt.mode}:${tt.slug}:${tt.label}:${tt.min ?? 0}:${tt.max ?? 120}`); + // The fallback must track MAX_TRAVEL_MINUTES (frontend/src/hooks/useTravelTime.ts). + // It read 120 against a live 90: dead, because every storyboard sets `max`, + // but the same stale number that made the commute drag land on 19 when the + // storyboard said 25. Reached, it would ask for a cap the dashboard clamps + // away, and the take would film a filter the storyboard never declared. + params.append('tt', `${tt.mode}:${tt.slug}:${tt.label}:${tt.min ?? 0}:${tt.max ?? TRAVEL_TIME_MAX_MIN}`); } - return `${APP_URL}${DASHBOARD_PATH}?${params}`; + return `${appUrl()}${DASHBOARD_PATH}?${params}`; } async function stubAiFilters(page: Page, storyboard: Storyboard) { diff --git a/video/src/runner.ts b/video/src/runner.ts index 3811739..a3e776b 100644 --- a/video/src/runner.ts +++ b/video/src/runner.ts @@ -1,7 +1,11 @@ -import { existsSync, readFileSync } from 'node:fs'; -import { join } from 'node:path'; import type { Page } from 'playwright'; -import { LEAD_IN_S, OUTPUT_DIR } from './config.js'; +import { LEAD_IN_S } from './config.js'; +import { + cueTiming, + loadSynthIndex, + overrunMessage, + type SynthCue, +} from './cue-timing.js'; import { clearVignette, hideAdScene, @@ -26,15 +30,6 @@ export interface RunnerResult { } const MAP_ZOOM_WHEEL_DELTA = -120; -const FALLBACK_MS_PER_WORD = 750; -const FALLBACK_TAIL_BUFFER_MS = 800; -const CJK_CHARS_PER_FALLBACK_WORD = 2; - -interface SynthCue { - cueIndex: number; - text: string; - durationMs: number; -} /** * Drive the recording from a cue-anchored storyboard. @@ -116,14 +111,11 @@ async function runCue( } const during = cue.during ?? []; - const declaredSum = during.reduce((s, a) => s + a.durationMs, 0); - if (declaredSum > measuredAudioMs + 50) { - throw new Error( - `Cue ${synth.cueIndex} "${cue.text.slice(0, 40)}…" has ${declaredSum}ms of ` + - `during activities but the measured audio is only ${measuredAudioMs}ms. ` + - `Trim a during step, lengthen the cue text, or move work into tail.` - ); - } + // Backstop only: dist/check-timing.js has already vetted every cue of every + // storyboard against this same rule, before recording started. Reaching a + // throw here means record.js was run outside render.sh. + const timing = cueTiming(cue, synth.cueIndex, measuredAudioMs); + if (timing.over) throw new Error(overrunMessage(cue, timing)); // Time the during block as a whole: individual steps may overrun their // budgets, but what matters at the cue boundary is total wall-clock. const duringStart = Date.now(); @@ -514,45 +506,4 @@ async function tryResolveTarget( } } -/** - * Load synth's measured cue durations. Falls back to a worst-case estimate - * if the manifest is missing: that path is only used for ``--no-audio`` - * runs, where the visual flow needs to play even without speech to time - * against. - */ -function loadSynthIndex(storyboard: Storyboard): SynthCue[] { - const path = join(OUTPUT_DIR, storyboard.name, 'audio', 'index.json'); - if (existsSync(path)) { - const raw = JSON.parse(readFileSync(path, 'utf-8')) as { - items: SynthCue[]; - }; - const byIndex = new Map(raw.items.map((it) => [it.cueIndex, it] as const)); - return storyboard.cues.map((cue, i) => { - const m = byIndex.get(i); - if (!m) { - throw new Error( - `Synth manifest is missing cue ${i} ("${cue.text.slice(0, 40)}…"). ` + - `Re-run preflight + synth so the audio matches the storyboard.` - ); - } - return m; - }); - } - console.log( - `[runner] no ${path} found. Using worst-case fallback durations (${FALLBACK_MS_PER_WORD}ms/word + ${FALLBACK_TAIL_BUFFER_MS}ms buffer). Audio will be missing.` - ); - return storyboard.cues.map((cue, cueIndex) => ({ - cueIndex, - text: cue.text, - durationMs: estimateFallbackDurationMs(cue.text), - })); -} - -function estimateFallbackDurationMs(text: string): number { - const wordCount = text.split(/\s+/).filter(Boolean).length; - const cjkCount = text.match(/\p{Script=Han}/gu)?.length ?? 0; - const units = Math.max(wordCount, Math.ceil(cjkCount / CJK_CHARS_PER_FALLBACK_WORD), 1); - return units * FALLBACK_MS_PER_WORD + FALLBACK_TAIL_BUFFER_MS; -} - export type { Page }; diff --git a/video/src/storyboard.ts b/video/src/storyboard.ts index 3d11b7f..a85236c 100644 --- a/video/src/storyboard.ts +++ b/video/src/storyboard.ts @@ -59,7 +59,14 @@ const SCHOOL_OUTSTANDING_PRIMARY = 'Outstanding primary school catchments'; const AI_ZOOM_SCALE_DESKTOP = 1.45; const TT_CARD_SELECTOR = '[data-filter-name="tt_0"]'; -const TT_SLIDER_MAX = 120; +// Must track MAX_TRAVEL_MINUTES in frontend/src/hooks/useTravelTime.ts, which +// is what TravelTimeCard hands the slider as its max. Only `toFraction` maths +// reads this, so drift does not throw: it silently drops the thumb at +// `target × (90/this)` of the way along and the take looks plausible. It was +// 120 against a live 90, which is why ttDragAct's comment below records a drop +// landing on 15 when the story said 20 (20/120 × 90 = 15 exactly). That was +// papered over with snapToValue rather than traced back to here. +const TT_SLIDER_MAX = 90; const TT_DRAG_FROM_MIN = 35; // 25 (not 20): tight enough that the drag visibly prunes the map, loose // enough that street-level central London keeps plenty of matching @@ -421,20 +428,34 @@ function createCues(locale: RecordingLocale, formFactor: FormFactor): Storyboard thumbSelector: `${TT_CARD_SELECTOR} [role="slider"] >> nth=1`, trackSelector: `${TT_CARD_SELECTOR} [data-orientation="horizontal"] >> nth=0`, toFraction: TT_DRAG_TO_MIN / TT_SLIDER_MAX, + // Land on the exact minute, like the ads' ttDragAct does. Without + // it this drag inherited the TT_SLIDER_MAX drift and dropped the + // thumb at 19, under the 20 that the TT_DRAG_TO_MIN note above + // records as emptying central London and leaving the postcode tap + // in the next cue with nothing to open. + snapToValue: TT_DRAG_TO_MIN, durationMs: 1600, }, { kind: 'click', target: colourTravelTime, durationMs: 700 }, ], - tail: [{ kind: 'wait', durationMs: 300 }], + // Clearing the sheet off the map and growing the cursor are staging for + // the zoom in the next cue, not part of any spoken line, so they run in + // this cue's tail. Inside the next cue's `during` they had to share the + // window with the zoom itself, and this cue builder is shared by all six + // locales: the budget has to fit the SHORTEST narration of the six (hi), + // not the one it was tuned against. + tail: [ + ...(isMobile + ? [{ kind: 'dragSheet', toHeightFrac: SHEET_DOWN, durationMs: 800 } as Activity] + : []), + { kind: 'cursorScale', scale: isMobile ? 1 : 1.4, durationMs: 200 }, + { kind: 'wait', durationMs: 300 }, + ], }, { text: copy.cues.streets, gapBeforeMs: 300, during: [ - ...(isMobile - ? [{ kind: 'dragSheet', toHeightFrac: SHEET_DOWN, durationMs: 800 } as Activity] - : []), - { kind: 'cursorScale', scale: isMobile ? 1 : 1.4, durationMs: 200 }, { // Zoom AROUND the strongest visible match (hex()) so the // destination area is guaranteed to contain matching postcodes. @@ -1120,12 +1141,15 @@ const AD_CONFIGS: DemoAdStoryboardConfig[] = [ text: 'Every colour on this map is a commute to central London.', caption: 'Same commute, cheaper postcode', during: [wait(400)], - tail: [wait(150)], + // Raise the sheet here rather than under the next line. Bringing the + // slider on screen is staging, and the next cue is a short one: with + // the sheet in it, the drag had to be rushed to fit the words. + tail: [wait(150), sheetUp(700)], }, { text: "Here's what twenty minutes actually leaves you.", caption: 'What 20 minutes buys', - during: [sheetUp(700), touchShow(), ttDragAct(20, 1700)], + during: [touchShow(), ttDragAct(20, 1700)], tail: [touchHide(), sheetDown(650)], }, { diff --git a/video/tts/mux.py b/video/tts/mux.py index 6855931..4a6253b 100644 --- a/video/tts/mux.py +++ b/video/tts/mux.py @@ -38,8 +38,20 @@ def measure_loudness(cmd_head: list[str], filter_complex: str) -> dict[str, str] Returns the measured values for the linear second pass, or None if the measurement failed (in which case the caller falls back to one-pass). + + loudnorm prints its JSON summary at ffmpeg's `info` level, so this pass + cannot inherit the caller's `-loglevel warning`: that suppressed the very + stats parsed below, the probe returned None every single time, and every + render silently took the one-pass fallback the docstring treats as the + exception. Nothing surfaced it because the fallback sounds fine until you + compare it: one-pass is the dynamic mode, which pumps on speech. """ - probe_cmd = cmd_head + [ + probe_head = list(cmd_head) + if "-loglevel" in probe_head: + probe_head[probe_head.index("-loglevel") + 1] = "info" + else: + probe_head[1:1] = ["-loglevel", "info"] + probe_cmd = probe_head + [ "-filter_complex", f"{filter_complex};[aout]loudnorm={LOUDNORM_TARGET}:print_format=json[anorm]", "-map", diff --git a/video/tts/synth.py b/video/tts/synth.py index ddba636..8b6f910 100644 --- a/video/tts/synth.py +++ b/video/tts/synth.py @@ -427,7 +427,7 @@ def resolve_voice(script_path: Path, block: dict) -> VoiceSettings: voice=voice, language=language, temperature=float(block.get("temperature", 0.8)), - exaggeration=float(block.get("exaggeration", 0.5)), + exaggeration=float(block.get("exaggeration", 0.65)), cfg_weight=float(block.get("cfgWeight", 0.5)), speed_factor=float(block.get("speedFactor", 1.0)), seed=int(block.get("seed", 42)),