From bf3f09aeb6c92976e4e7e0b45cf6fdb09f8f1d9e Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Tue, 14 Jul 2026 19:28:09 +0100 Subject: [PATCH] Fix render --- video/render.sh | 118 ++++++++++++++++++--------------------- video/tts/pyproject.toml | 11 +++- video/tts/synth.py | 41 +++++++++++--- video/tts/uv.lock | 6 -- 4 files changed, 96 insertions(+), 80 deletions(-) diff --git a/video/render.sh b/video/render.sh index e9812bd..dbf6ab5 100755 --- a/video/render.sh +++ b/video/render.sh @@ -22,9 +22,10 @@ # ./render.sh --fresh-auth # force re-auth even if cache is fresh # ./render.sh --resume # preserve completed recordings and continue # ./render.sh --no-encode # stop at WebM, skip MP4 encode -# ./render.sh --no-audio # skip Qwen3-TTS narration; silent MP4 +# ./render.sh --no-audio # skip narration synthesis; silent MP4 # FORCE_AUTH=1 ./render.sh # same as --fresh-auth # APP_URL=http://localhost:3001 ./render.sh # override frontend URL +# CHATTERBOX_URL=http://box:8004 ./render.sh # override the TTS server # # Cred env vars (read for both targets, but prod has no fallback defaults): # LOGIN_EMAIL, LOGIN_PASSWORD: the dashboard account to record as @@ -80,6 +81,9 @@ else fi export PB_BOOTSTRAP_ADMIN export LOGIN_EMAIL LOGIN_PASSWORD +# Narration TTS. Not target-specific: Chatterbox is a LAN service, so prod +# renders and local renders both narrate from the same box. +export CHATTERBOX_URL="${CHATTERBOX_URL:-http://host.docker.internal:8004}" # Per-target storage state: switching targets must not reuse a stale token. # config.ts reads AUTH_STATE_FILE for AUTH_STATE_PATH. export AUTH_STATE_FILE="${AUTH_STATE_FILE:-auth.${TARGET}.json}" @@ -101,7 +105,10 @@ for arg in "${@:-}"; do --no-encode) DO_ENCODE=0 ;; --no-audio) DO_AUDIO=0 ;; -h|--help) - sed -n '3,32p' "$0" + # Print the header comment block, stopping at the first non-comment + # line. A hardcoded end line silently truncates --help mid-sentence the + # next time someone documents a flag. + sed -n '3,/^[^#]/p' "$0" | sed '$d' exit 0 ;; *) echo "Unknown arg: $arg" >&2; exit 2 ;; esac @@ -128,6 +135,29 @@ http_code() { curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 --max-time 5 "$1" || echo "000" } +# Order storyboards so all English cuts synth together and all non-English +# cuts synth together. The Chatterbox server holds one checkpoint at a time +# and English lives on a different one from every other language, so an +# interleaved order (recording, recording-de, recording-zh, recording-hi) +# would pay a ~20s swap three times instead of once. Reads the language each +# storyboard declared in the narration script preflight just emitted; any +# storyboard whose script is unreadable sorts last and still gets rendered. +synth_order() { + local sb lang + for sb in "$@"; do + lang="$(node -e ' + const fs = require("node:fs"); + try { + process.stdout.write( + JSON.parse(fs.readFileSync(process.argv[1], "utf-8")).voice?.language ?? "zz" + ); + } catch { process.stdout.write("zz"); } + ' "output/$sb/narration-script.json" 2>/dev/null || echo "zz")" + # en sorts before everything else; the rest group by language code. + printf '%s\t%s\n' "$([ "$lang" = "en" ] && echo "0-en" || echo "1-$lang")" "$sb" + done | sort -s -k1,1 | cut -f2 +} + wait_for() { local url="$1" desc="$2" timeout="${3:-90}" say "Waiting for $desc ($url)" @@ -292,80 +322,40 @@ for sb in "${STORYBOARDS[@]}"; do fi done -# -- synth (Qwen3-TTS) ------------------------------------------------------- -# Synth runs BEFORE recording: one batched generate_voice_clone call per -# storyboard so the voice stays consistent within each video. The recorder -# reads output//audio/index.json for measured per-cue durations and -# sizes each cue's wall-clock to fit; --no-audio skips synth and the recorder -# falls back to a worst-case estimate. +# -- synth (Chatterbox) ------------------------------------------------------ +# Synth runs BEFORE recording: the recorder reads output//audio/index.json +# for measured per-cue durations and sizes each cue's wall-clock to fit; +# --no-audio skips synth and the recorder falls back to a worst-case estimate. +# +# Narration is generated by the Chatterbox TTS server over HTTP, so there is +# nothing to install and no GPU to schedule here. Voice timbre needs no +# defending either: a predefined voice is a fixed asset on the server, so +# every cue of every storyboard clones from identical bytes by construction. +# (The Qwen3-TTS pipeline this replaced had to mint a reference WAV per render +# and hand-copy it between storyboards to stop two voices landing in one cut.) +# +# Storyboards are synthesised in language order so an en/de/zh set pays the +# ~20s English<->multilingual checkpoint swap once instead of once per cut. +# synth.py itself only swaps when the server is on the wrong checkpoint. if [ "$DO_AUDIO" = "1" ]; then if ! command -v uv >/dev/null 2>&1; then - fail "uv not on PATH (required for Qwen3-TTS synth). Install uv or rerun with --no-audio." - fi - # The torch/cudnn wheels are ~700MB; uv's 30s default chokes on first sync. - export UV_HTTP_TIMEOUT="${UV_HTTP_TIMEOUT:-600}" - # Pull in the flash-attn prebuilt wheel (defined as the `gpu` extra) when - # the host actually has a GPU. The wheel is bound to torch 2.6 + cu12 + - # cp312. See tts/pyproject.toml. - uv_sync_extras=() - if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then - uv_sync_extras+=(--extra gpu) + fail "uv not on PATH (required for tts/synth.py). Install uv or rerun with --no-audio." fi say "Synchronising tts/ Python deps" - uv sync --project tts ${uv_sync_extras[@]+"${uv_sync_extras[@]}"} || fail "uv sync failed in video/tts" + uv sync --project tts || fail "uv sync failed in video/tts" - if [ -z "${TTS_DEVICE:-}" ]; then - if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then - gpu_free_mb="$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')" - if [ "${gpu_free_mb:-0}" -ge 8000 ]; then - export TTS_DEVICE="cuda:0" - else - export TTS_DEVICE="cpu" - say "GPU has ${gpu_free_mb:-0}MiB free; using CPU for TTS to avoid CUDA OOM" - fi - else - export TTS_DEVICE="cpu" - fi + cb_code="$(http_code "$CHATTERBOX_URL/api/model-info")" + if [ "$cb_code" != "200" ]; then + fail "Cannot reach the Chatterbox TTS server at $CHATTERBOX_URL (got $cb_code). Start it, set CHATTERBOX_URL, or rerun with --no-audio." fi - # Voice consistency: every ad in this set declares the same AD_VOICE - # (instruct/seed/temperature/topP/referenceText). Even with seed-locked - # VoiceDesign, independent invocations across processes can produce - # mildly different reference waveforms, different enough that a - # listener notices the timbre shift across ads. To avoid that, we - # mint the reference WAV ONCE (from the first storyboard) and reuse - # it across the rest of the storyboards by copying _reference.wav + - # _reference.meta.json into their audio dirs before their synth runs. - # synth.py's _resolve_reference() reuses a matching cached reference - # as long as the meta block (instruct/language/seed/etc.) matches. - # - # We copy ONLY the reference, never the cue wavs or index.json. Copying - # the whole audio dir (as an earlier version did) overwrote each later - # storyboard's cached index.json with the FIRST storyboard's, which - # forced a full re-synth on every run, and in multi-voice sets (the - # localized homepage demos: en/de/zh/hi) it clobbered correct localized - # audio. With a reference-only copy: same-voice sets reuse the reference - # (meta matches); different-voice sets re-mint their own (meta mismatch), - # and in both cases an up-to-date cached index.json lets synth skip. - shared_ref_wav="" - shared_ref_meta="" - for sb in "${STORYBOARDS[@]}"; do - if [ -n "$shared_ref_wav" ] && [ -f "$shared_ref_wav" ] && [ -f "$shared_ref_meta" ]; then - mkdir -p "output/$sb/audio" - cp -f "$shared_ref_wav" "output/$sb/audio/_reference.wav" - cp -f "$shared_ref_meta" "output/$sb/audio/_reference.meta.json" - fi + for sb in $(synth_order "${STORYBOARDS[@]}"); do say "Synthesising narration for [$sb]" - uv run --project tts python tts/synth.py --storyboard "$sb" \ + CHATTERBOX_URL="$CHATTERBOX_URL" uv run --project tts python tts/synth.py --storyboard "$sb" \ || fail "tts/synth.py failed for $sb" if [ ! -s "output/$sb/audio/index.json" ]; then fail "synth did not produce output/$sb/audio/index.json" fi - if [ -z "$shared_ref_wav" ] && [ -f "output/$sb/audio/_reference.wav" ]; then - shared_ref_wav="output/$sb/audio/_reference.wav" - shared_ref_meta="output/$sb/audio/_reference.meta.json" - say "Locked voice reference to $shared_ref_wav; reusing for the rest of the set" - fi done fi diff --git a/video/tts/pyproject.toml b/video/tts/pyproject.toml index f4d82bb..06de6a0 100644 --- a/video/tts/pyproject.toml +++ b/video/tts/pyproject.toml @@ -9,7 +9,12 @@ requires-python = ">=3.12" # model in-process, which dragged in torch + torchaudio + a CUDA-matched # flash-attn wheel (~700MB of downloads, pinned to the host's driver version # and to cp312). Generating over HTTP moved all of that onto the server. +# +# Nothing here pins a platform or an interpreter ceiling on purpose: the old +# `[tool.uv] environments` marker existed only to keep torch/torchaudio/ +# flash-attn resolving to linux + cp312. Left in place it would outlive its +# reason and break the build, because it caps the LOCK at python < 3.14 while +# requires-python lets uv pick a newer interpreter: `uv sync` then fails with +# "the current Python platform is not compatible with the lockfile's supported +# environments" and render.sh aborts before synthesising a single cue. dependencies = [] - -[tool.uv] -environments = ["sys_platform == 'linux' and python_version < '3.14'"] diff --git a/video/tts/synth.py b/video/tts/synth.py index 29de333..4e5a7cc 100644 --- a/video/tts/synth.py +++ b/video/tts/synth.py @@ -283,17 +283,34 @@ def ramble_baseline(items: list[dict]) -> float | None: def rambling_cues(items: list[dict]) -> list[dict]: - """Cues running far enough past the storyboard's pace to look hallucinated.""" + """Cues running far enough past the storyboard's pace to look hallucinated. + + Cues already marked `rambleRepairExhausted` are excluded: we tried every + re-roll and kept the best take we could get, so re-flagging them would + re-run the identical seeds (the server is deterministic per seed) for the + identical result on every render, and would defeat the cache forever. + """ baseline = ramble_baseline(items) if baseline is None: return [] return [ it for it in items - if secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE + if not it.get("rambleRepairExhausted") + and secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE ] +def within_pace(rate: float, baseline: float) -> bool: + """Is this take's speech rate sane, in EITHER direction? + + Two-sided on purpose. A one-sided "not too long" gate would accept a take + whose ending was cut off (rate far BELOW baseline) and stop re-rolling, + quietly shipping clipped narration in place of a ramble. + """ + return baseline / RAMBLE_TOLERANCE <= rate <= baseline * RAMBLE_TOLERANCE + + def repair_rambling_cues( server: str, audio_dir: Path, @@ -332,12 +349,17 @@ def repair_rambling_cues( error = abs(candidate_rate - baseline) if error < best_error: best, best_error = (data, sample_rate, duration_ms, seed), error - if candidate_rate <= baseline * RAMBLE_TOLERANCE: + if within_pace(candidate_rate, baseline): break if best is None: + # Every re-roll was worse than the take we already had. Record that + # we exhausted the sweep so the next render trusts the cache + # instead of re-running these exact seeds for these exact results. + item["rambleRepairExhausted"] = True print( - f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it", + f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it. " + "Listen before publishing.", flush=True, ) continue @@ -347,11 +369,16 @@ def repair_rambling_cues( item["durationMs"] = duration_ms item["seed"] = seed repaired += 1 - settled = secs_per_char(item["text"], duration_ms) / baseline - note = "" if settled <= RAMBLE_TOLERANCE else " STILL LONG: listen before publishing" + settled = secs_per_char(item["text"], duration_ms) + note = "" + if not within_pace(settled, baseline): + # Improved but still off-pace: same reasoning as above, stop here + # rather than re-litigating it on every render. + item["rambleRepairExhausted"] = True + note = " STILL OFF-PACE: listen before publishing" print( f"[synth] cue {item['cueIndex']}: {duration_ms}ms on seed {seed} " - f"({settled:.1f}x pace){note}", + f"({settled / baseline:.1f}x pace){note}", flush=True, ) return repaired diff --git a/video/tts/uv.lock b/video/tts/uv.lock index a68bf89..14fd992 100644 --- a/video/tts/uv.lock +++ b/video/tts/uv.lock @@ -1,12 +1,6 @@ version = 1 revision = 3 requires-python = ">=3.12" -resolution-markers = [ - "python_full_version < '3.14' and sys_platform == 'linux'", -] -supported-markers = [ - "python_full_version < '3.14' and sys_platform == 'linux'", -] [[package]] name = "property-map-video-tts"