Fix render
This commit is contained in:
parent
68097386df
commit
bf3f09aeb6
4 changed files with 96 additions and 80 deletions
118
video/render.sh
118
video/render.sh
|
|
@ -22,9 +22,10 @@
|
|||
# ./render.sh --fresh-auth # force re-auth even if cache is fresh
|
||||
# ./render.sh --resume # preserve completed recordings and continue
|
||||
# ./render.sh --no-encode # stop at WebM, skip MP4 encode
|
||||
# ./render.sh --no-audio # skip Qwen3-TTS narration; silent MP4
|
||||
# ./render.sh --no-audio # skip narration synthesis; silent MP4
|
||||
# FORCE_AUTH=1 ./render.sh # same as --fresh-auth
|
||||
# APP_URL=http://localhost:3001 ./render.sh # override frontend URL
|
||||
# CHATTERBOX_URL=http://box:8004 ./render.sh # override the TTS server
|
||||
#
|
||||
# Cred env vars (read for both targets, but prod has no fallback defaults):
|
||||
# LOGIN_EMAIL, LOGIN_PASSWORD: the dashboard account to record as
|
||||
|
|
@ -80,6 +81,9 @@ else
|
|||
fi
|
||||
export PB_BOOTSTRAP_ADMIN
|
||||
export LOGIN_EMAIL LOGIN_PASSWORD
|
||||
# Narration TTS. Not target-specific: Chatterbox is a LAN service, so prod
|
||||
# renders and local renders both narrate from the same box.
|
||||
export CHATTERBOX_URL="${CHATTERBOX_URL:-http://host.docker.internal:8004}"
|
||||
# Per-target storage state: switching targets must not reuse a stale token.
|
||||
# config.ts reads AUTH_STATE_FILE for AUTH_STATE_PATH.
|
||||
export AUTH_STATE_FILE="${AUTH_STATE_FILE:-auth.${TARGET}.json}"
|
||||
|
|
@ -101,7 +105,10 @@ for arg in "${@:-}"; do
|
|||
--no-encode) DO_ENCODE=0 ;;
|
||||
--no-audio) DO_AUDIO=0 ;;
|
||||
-h|--help)
|
||||
sed -n '3,32p' "$0"
|
||||
# Print the header comment block, stopping at the first non-comment
|
||||
# line. A hardcoded end line silently truncates --help mid-sentence the
|
||||
# next time someone documents a flag.
|
||||
sed -n '3,/^[^#]/p' "$0" | sed '$d'
|
||||
exit 0 ;;
|
||||
*) echo "Unknown arg: $arg" >&2; exit 2 ;;
|
||||
esac
|
||||
|
|
@ -128,6 +135,29 @@ http_code() {
|
|||
curl -s -o /dev/null -w "%{http_code}" --connect-timeout 3 --max-time 5 "$1" || echo "000"
|
||||
}
|
||||
|
||||
# Order storyboards so all English cuts synth together and all non-English
|
||||
# cuts synth together. The Chatterbox server holds one checkpoint at a time
|
||||
# and English lives on a different one from every other language, so an
|
||||
# interleaved order (recording, recording-de, recording-zh, recording-hi)
|
||||
# would pay a ~20s swap three times instead of once. Reads the language each
|
||||
# storyboard declared in the narration script preflight just emitted; any
|
||||
# storyboard whose script is unreadable sorts last and still gets rendered.
|
||||
synth_order() {
|
||||
local sb lang
|
||||
for sb in "$@"; do
|
||||
lang="$(node -e '
|
||||
const fs = require("node:fs");
|
||||
try {
|
||||
process.stdout.write(
|
||||
JSON.parse(fs.readFileSync(process.argv[1], "utf-8")).voice?.language ?? "zz"
|
||||
);
|
||||
} catch { process.stdout.write("zz"); }
|
||||
' "output/$sb/narration-script.json" 2>/dev/null || echo "zz")"
|
||||
# en sorts before everything else; the rest group by language code.
|
||||
printf '%s\t%s\n' "$([ "$lang" = "en" ] && echo "0-en" || echo "1-$lang")" "$sb"
|
||||
done | sort -s -k1,1 | cut -f2
|
||||
}
|
||||
|
||||
wait_for() {
|
||||
local url="$1" desc="$2" timeout="${3:-90}"
|
||||
say "Waiting for $desc ($url)"
|
||||
|
|
@ -292,80 +322,40 @@ for sb in "${STORYBOARDS[@]}"; do
|
|||
fi
|
||||
done
|
||||
|
||||
# -- synth (Qwen3-TTS) -------------------------------------------------------
|
||||
# Synth runs BEFORE recording: one batched generate_voice_clone call per
|
||||
# storyboard so the voice stays consistent within each video. The recorder
|
||||
# reads output/<sb>/audio/index.json for measured per-cue durations and
|
||||
# sizes each cue's wall-clock to fit; --no-audio skips synth and the recorder
|
||||
# falls back to a worst-case estimate.
|
||||
# -- synth (Chatterbox) ------------------------------------------------------
|
||||
# Synth runs BEFORE recording: the recorder reads output/<sb>/audio/index.json
|
||||
# for measured per-cue durations and sizes each cue's wall-clock to fit;
|
||||
# --no-audio skips synth and the recorder falls back to a worst-case estimate.
|
||||
#
|
||||
# Narration is generated by the Chatterbox TTS server over HTTP, so there is
|
||||
# nothing to install and no GPU to schedule here. Voice timbre needs no
|
||||
# defending either: a predefined voice is a fixed asset on the server, so
|
||||
# every cue of every storyboard clones from identical bytes by construction.
|
||||
# (The Qwen3-TTS pipeline this replaced had to mint a reference WAV per render
|
||||
# and hand-copy it between storyboards to stop two voices landing in one cut.)
|
||||
#
|
||||
# Storyboards are synthesised in language order so an en/de/zh set pays the
|
||||
# ~20s English<->multilingual checkpoint swap once instead of once per cut.
|
||||
# synth.py itself only swaps when the server is on the wrong checkpoint.
|
||||
if [ "$DO_AUDIO" = "1" ]; then
|
||||
if ! command -v uv >/dev/null 2>&1; then
|
||||
fail "uv not on PATH (required for Qwen3-TTS synth). Install uv or rerun with --no-audio."
|
||||
fi
|
||||
# The torch/cudnn wheels are ~700MB; uv's 30s default chokes on first sync.
|
||||
export UV_HTTP_TIMEOUT="${UV_HTTP_TIMEOUT:-600}"
|
||||
# Pull in the flash-attn prebuilt wheel (defined as the `gpu` extra) when
|
||||
# the host actually has a GPU. The wheel is bound to torch 2.6 + cu12 +
|
||||
# cp312. See tts/pyproject.toml.
|
||||
uv_sync_extras=()
|
||||
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
|
||||
uv_sync_extras+=(--extra gpu)
|
||||
fail "uv not on PATH (required for tts/synth.py). Install uv or rerun with --no-audio."
|
||||
fi
|
||||
say "Synchronising tts/ Python deps"
|
||||
uv sync --project tts ${uv_sync_extras[@]+"${uv_sync_extras[@]}"} || fail "uv sync failed in video/tts"
|
||||
uv sync --project tts || fail "uv sync failed in video/tts"
|
||||
|
||||
if [ -z "${TTS_DEVICE:-}" ]; then
|
||||
if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L >/dev/null 2>&1; then
|
||||
gpu_free_mb="$(nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits 2>/dev/null | head -1 | tr -d ' ')"
|
||||
if [ "${gpu_free_mb:-0}" -ge 8000 ]; then
|
||||
export TTS_DEVICE="cuda:0"
|
||||
else
|
||||
export TTS_DEVICE="cpu"
|
||||
say "GPU has ${gpu_free_mb:-0}MiB free; using CPU for TTS to avoid CUDA OOM"
|
||||
fi
|
||||
else
|
||||
export TTS_DEVICE="cpu"
|
||||
fi
|
||||
cb_code="$(http_code "$CHATTERBOX_URL/api/model-info")"
|
||||
if [ "$cb_code" != "200" ]; then
|
||||
fail "Cannot reach the Chatterbox TTS server at $CHATTERBOX_URL (got $cb_code). Start it, set CHATTERBOX_URL, or rerun with --no-audio."
|
||||
fi
|
||||
|
||||
# Voice consistency: every ad in this set declares the same AD_VOICE
|
||||
# (instruct/seed/temperature/topP/referenceText). Even with seed-locked
|
||||
# VoiceDesign, independent invocations across processes can produce
|
||||
# mildly different reference waveforms, different enough that a
|
||||
# listener notices the timbre shift across ads. To avoid that, we
|
||||
# mint the reference WAV ONCE (from the first storyboard) and reuse
|
||||
# it across the rest of the storyboards by copying _reference.wav +
|
||||
# _reference.meta.json into their audio dirs before their synth runs.
|
||||
# synth.py's _resolve_reference() reuses a matching cached reference
|
||||
# as long as the meta block (instruct/language/seed/etc.) matches.
|
||||
#
|
||||
# We copy ONLY the reference, never the cue wavs or index.json. Copying
|
||||
# the whole audio dir (as an earlier version did) overwrote each later
|
||||
# storyboard's cached index.json with the FIRST storyboard's, which
|
||||
# forced a full re-synth on every run, and in multi-voice sets (the
|
||||
# localized homepage demos: en/de/zh/hi) it clobbered correct localized
|
||||
# audio. With a reference-only copy: same-voice sets reuse the reference
|
||||
# (meta matches); different-voice sets re-mint their own (meta mismatch),
|
||||
# and in both cases an up-to-date cached index.json lets synth skip.
|
||||
shared_ref_wav=""
|
||||
shared_ref_meta=""
|
||||
for sb in "${STORYBOARDS[@]}"; do
|
||||
if [ -n "$shared_ref_wav" ] && [ -f "$shared_ref_wav" ] && [ -f "$shared_ref_meta" ]; then
|
||||
mkdir -p "output/$sb/audio"
|
||||
cp -f "$shared_ref_wav" "output/$sb/audio/_reference.wav"
|
||||
cp -f "$shared_ref_meta" "output/$sb/audio/_reference.meta.json"
|
||||
fi
|
||||
for sb in $(synth_order "${STORYBOARDS[@]}"); do
|
||||
say "Synthesising narration for [$sb]"
|
||||
uv run --project tts python tts/synth.py --storyboard "$sb" \
|
||||
CHATTERBOX_URL="$CHATTERBOX_URL" uv run --project tts python tts/synth.py --storyboard "$sb" \
|
||||
|| fail "tts/synth.py failed for $sb"
|
||||
if [ ! -s "output/$sb/audio/index.json" ]; then
|
||||
fail "synth did not produce output/$sb/audio/index.json"
|
||||
fi
|
||||
if [ -z "$shared_ref_wav" ] && [ -f "output/$sb/audio/_reference.wav" ]; then
|
||||
shared_ref_wav="output/$sb/audio/_reference.wav"
|
||||
shared_ref_meta="output/$sb/audio/_reference.meta.json"
|
||||
say "Locked voice reference to $shared_ref_wav; reusing for the rest of the set"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,12 @@ requires-python = ">=3.12"
|
|||
# model in-process, which dragged in torch + torchaudio + a CUDA-matched
|
||||
# flash-attn wheel (~700MB of downloads, pinned to the host's driver version
|
||||
# and to cp312). Generating over HTTP moved all of that onto the server.
|
||||
#
|
||||
# Nothing here pins a platform or an interpreter ceiling on purpose: the old
|
||||
# `[tool.uv] environments` marker existed only to keep torch/torchaudio/
|
||||
# flash-attn resolving to linux + cp312. Left in place it would outlive its
|
||||
# reason and break the build, because it caps the LOCK at python < 3.14 while
|
||||
# requires-python lets uv pick a newer interpreter: `uv sync` then fails with
|
||||
# "the current Python platform is not compatible with the lockfile's supported
|
||||
# environments" and render.sh aborts before synthesising a single cue.
|
||||
dependencies = []
|
||||
|
||||
[tool.uv]
|
||||
environments = ["sys_platform == 'linux' and python_version < '3.14'"]
|
||||
|
|
|
|||
|
|
@ -283,17 +283,34 @@ def ramble_baseline(items: list[dict]) -> float | None:
|
|||
|
||||
|
||||
def rambling_cues(items: list[dict]) -> list[dict]:
|
||||
"""Cues running far enough past the storyboard's pace to look hallucinated."""
|
||||
"""Cues running far enough past the storyboard's pace to look hallucinated.
|
||||
|
||||
Cues already marked `rambleRepairExhausted` are excluded: we tried every
|
||||
re-roll and kept the best take we could get, so re-flagging them would
|
||||
re-run the identical seeds (the server is deterministic per seed) for the
|
||||
identical result on every render, and would defeat the cache forever.
|
||||
"""
|
||||
baseline = ramble_baseline(items)
|
||||
if baseline is None:
|
||||
return []
|
||||
return [
|
||||
it
|
||||
for it in items
|
||||
if secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE
|
||||
if not it.get("rambleRepairExhausted")
|
||||
and secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE
|
||||
]
|
||||
|
||||
|
||||
def within_pace(rate: float, baseline: float) -> bool:
|
||||
"""Is this take's speech rate sane, in EITHER direction?
|
||||
|
||||
Two-sided on purpose. A one-sided "not too long" gate would accept a take
|
||||
whose ending was cut off (rate far BELOW baseline) and stop re-rolling,
|
||||
quietly shipping clipped narration in place of a ramble.
|
||||
"""
|
||||
return baseline / RAMBLE_TOLERANCE <= rate <= baseline * RAMBLE_TOLERANCE
|
||||
|
||||
|
||||
def repair_rambling_cues(
|
||||
server: str,
|
||||
audio_dir: Path,
|
||||
|
|
@ -332,12 +349,17 @@ def repair_rambling_cues(
|
|||
error = abs(candidate_rate - baseline)
|
||||
if error < best_error:
|
||||
best, best_error = (data, sample_rate, duration_ms, seed), error
|
||||
if candidate_rate <= baseline * RAMBLE_TOLERANCE:
|
||||
if within_pace(candidate_rate, baseline):
|
||||
break
|
||||
|
||||
if best is None:
|
||||
# Every re-roll was worse than the take we already had. Record that
|
||||
# we exhausted the sweep so the next render trusts the cache
|
||||
# instead of re-running these exact seeds for these exact results.
|
||||
item["rambleRepairExhausted"] = True
|
||||
print(
|
||||
f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it",
|
||||
f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it. "
|
||||
"Listen before publishing.",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
|
|
@ -347,11 +369,16 @@ def repair_rambling_cues(
|
|||
item["durationMs"] = duration_ms
|
||||
item["seed"] = seed
|
||||
repaired += 1
|
||||
settled = secs_per_char(item["text"], duration_ms) / baseline
|
||||
note = "" if settled <= RAMBLE_TOLERANCE else " STILL LONG: listen before publishing"
|
||||
settled = secs_per_char(item["text"], duration_ms)
|
||||
note = ""
|
||||
if not within_pace(settled, baseline):
|
||||
# Improved but still off-pace: same reasoning as above, stop here
|
||||
# rather than re-litigating it on every render.
|
||||
item["rambleRepairExhausted"] = True
|
||||
note = " STILL OFF-PACE: listen before publishing"
|
||||
print(
|
||||
f"[synth] cue {item['cueIndex']}: {duration_ms}ms on seed {seed} "
|
||||
f"({settled:.1f}x pace){note}",
|
||||
f"({settled / baseline:.1f}x pace){note}",
|
||||
flush=True,
|
||||
)
|
||||
return repaired
|
||||
|
|
|
|||
6
video/tts/uv.lock
generated
6
video/tts/uv.lock
generated
|
|
@ -1,12 +1,6 @@
|
|||
version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.12"
|
||||
resolution-markers = [
|
||||
"python_full_version < '3.14' and sys_platform == 'linux'",
|
||||
]
|
||||
supported-markers = [
|
||||
"python_full_version < '3.14' and sys_platform == 'linux'",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "property-map-video-tts"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue