andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

migrate to chatterbox

Browse source
This commit is contained in:
Andras Schmelczer 2026-07-14 16:59:48 +01:00
parent 716e42a57d
commit 988c01c4f7
7 changed files with 480 additions and 1786 deletions
Download patch file Download diff file Expand all files Collapse all files

4
video/tts/mux.py
View file

@ -3,7 +3,7 @@
Reads two manifests inside ``output/<storyboard>/``:
* ``audio/index.json`` (synth output): per-cue WAV filename + measured
duration. Generated BEFORE recording in one batched Qwen3-TTS call.
duration. Generated BEFORE recording by ``tts/synth.py``.
* ``narration.json`` (recorder output): per-cue ``videoTimeMs`` against
the trimmed video. Generated DURING recording.
@ -30,7 +30,7 @@ from pathlib import Path
# Mixing the narration WAVs raw left the muxed track at ~-24 LUFS: viewers
# had to crank the volume, which reads as low production quality. Target a
# touch under the platform norm so speech-only audio never pumps.
LOUDNORM_TARGET = "I=-15:TP=-1.5:LRA=11"
LOUDNORM_TARGET = "I=-14:TP=-1.5:LRA=11"
def measure_loudness(cmd_head: list[str], filter_complex: str) -> dict[str, str] | None:

46
video/tts/pyproject.toml
View file

@ -1,41 +1,15 @@
[project]
name = "property-map-video-tts"
version = "0.1.0"
description = "Qwen3-TTS narration generator for the homepage demo video."
requires-python = ">=3.12,<3.13"
dependencies = [
"qwen-tts>=0.1.1",
# Host driver is CUDA 12.4 (see `nvidia-smi`). torch 2.7+ dropped cu124
# wheels, so we cap below that and pull the cu124 build from PyTorch's
# own index (configured below). torchaudio must match torch's CUDA build:
# the PyPI default ships a CUDA 13 binary that fails to load
# libcudart.so.13 on this host.
"torch>=2.5,<2.7",
"torchaudio>=2.5,<2.7",
"soundfile>=0.12",
"numpy>=1.26",
]
[project.optional-dependencies]
# Flash-attention prebuilt wheel matched to torch 2.6 + cu12 + cp312, old
# CXX ABI (PyTorch's cu124 wheel reports compiled_with_cxx11_abi() == False
# and only exports the old-ABI c10::Error constructor). Pinned to
# 2.7.4.post1 because 2.8.x's torch2.6/abiFALSE wheels were mislabelled:
# they ship new-ABI symbols and fail to import. Building from source needs
# nvcc which isn't on the host. Enable via `uv sync --extra gpu`; render.sh
# does this automatically when nvidia-smi reports a GPU.
gpu = [
"flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1%2Bcu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
]
description = "Chatterbox narration generator + ffmpeg muxer for the homepage demo video."
requires-python = ">=3.12"
# Deliberately dependency-free. synth.py talks to the Chatterbox TTS server
# over HTTP (urllib) and reads WAV headers with the stdlib `wave` module;
# mux.py shells out to ffmpeg. The previous Qwen3-TTS implementation ran the
# model in-process, which dragged in torch + torchaudio + a CUDA-matched
# flash-attn wheel (~700MB of downloads, pinned to the host's driver version
# and to cp312). Generating over HTTP moved all of that onto the server.
dependencies = []
[tool.uv]
environments = ["sys_platform == 'linux' and python_version < '3.13'"]
[tool.uv.sources]
torch = [{ index = "pytorch-cu124" }]
torchaudio = [{ index = "pytorch-cu124" }]
[[tool.uv.index]]
name = "pytorch-cu124"
url = "https://download.pytorch.org/whl/cu124"
explicit = true
environments = ["sys_platform == 'linux' and python_version < '3.14'"]

787
video/tts/synth.py
View file

@ -1,26 +1,34 @@
"""Synthesize one storyboard's narration in ONE batched Qwen3-TTS call.
"""Synthesize one storyboard's narration with the Chatterbox TTS server.
Reads ``output/<storyboard>/narration-script.json`` (emitted by
``dist/preflight.js``) and runs ``Qwen3TTSModel.generate_voice_design`` with
all cue texts as a single batched list: that way every cue shares the same
model state, which keeps prosody and timbre consistent across cues. Per-cue
WAVs and an index manifest go to ``output/<storyboard>/audio/`` for the
recording step (which reads measured cue durations) and the mux step (which
drops each WAV at its videoTime).
``dist/preflight.js``) and POSTs one cue at a time to the Chatterbox server's
native ``/tts`` endpoint. Per-cue WAVs and an index manifest go to
``output/<storyboard>/audio/`` for the recording step (which reads measured
cue durations) and the mux step (which drops each WAV at its videoTime).
Voice persona, language, and sampling come from the storyboard via the
``voice`` block of the narration script. CLI flags can still override them
for ad-hoc experimentation; storyboards remain the source of truth for
production runs.
Voice, language and sampling come from the storyboard via the ``voice`` block
of the narration script.
We use the VoiceDesign sibling of CustomVoice because it accepts a free-form
voice persona (British accent, narrator register, "no laughter") via the
``instruct`` parameter. CustomVoice's preset speakers are all American or
non-English, and its ``instruct`` is documented for emotion only. It
ignored accent directives and bled non-speech tokens (laughter, sighs)
between cues.
Chatterbox ships two checkpoints and the server holds exactly ONE at a time:
Run from the ``video/`` directory:
* ``chatterbox`` (reported by the server as type ``original``) is English-only.
* ``multilingual`` covers 23 languages, German and Chinese among them.
We pick the checkpoint from ``voice.language`` and hot-swap the server when it
is on the wrong one. The swap is load-bearing, not cosmetic: the original
checkpoint happily ACCEPTS ``language: "de"`` and returns HTTP 200, but reads
the text with an English frontend, so a missed swap yields confident-sounding
rubbish instead of an error. Nothing downstream would catch that, hence
``ensure_model``.
Unlike the Qwen3-TTS pipeline this replaces, there is no voice-design or
reference-cloning stage: a Chatterbox predefined voice IS the reference, it
lives on the server, and it is byte-identical for every cue and every
storyboard. That deletes the whole minted-reference/referenceHash machinery
along with the "two voices in one video" class of bug it existed to prevent.
Run from the ``video/`` directory (the server must already be reachable;
render.sh health-checks it first):
uv run --project tts python tts/synth.py --storyboard recording
"""
@ -28,60 +36,101 @@ Run from the ``video/`` directory:
from __future__ import annotations
import argparse
import hashlib
import io
import json
import os
import random
import statistics
import sys
import time
import urllib.error
import urllib.request
import wave
from dataclasses import dataclass
from pathlib import Path
import numpy as np
import soundfile as sf
import torch
from qwen_tts import Qwen3TTSModel
DEFAULT_SERVER = "http://host.docker.internal:8004"
# repo_id we POST to /save_settings -> the `type` /api/model-info reports back.
# The two vocabularies differ, so we cannot compare them directly.
ENGLISH_REPO_ID = "chatterbox"
MULTILINGUAL_REPO_ID = "multilingual"
MODEL_TYPE_BY_REPO_ID = {ENGLISH_REPO_ID: "original", MULTILINGUAL_REPO_ID: "multilingual"}
# Cues are one or two sentences (the longest across every storyboard is ~105
# chars), so we synthesize each in a single pass: server-side splitting would
# only add chunk seams inside a cue for no benefit. If narration ever grows
# past this, revisit rather than silently degrade.
SPLIT_TEXT_WARN_CHARS = 300
# A hot-swap unloads the current checkpoint, clears VRAM and loads the other
# one; it took ~22s on the dev box. Generation itself is ~3s per cue.
SWAP_TIMEOUT_S = 300
GENERATE_TIMEOUT_S = 300
INFO_TIMEOUT_S = 30
GENERATE_ATTEMPTS = 3
RETRY_BACKOFF_S = 2.0
# Chatterbox sometimes keeps going after it has finished the line: the cue's
# words all arrive, then several seconds of invented babble follow. It is a
# property of the seed, not the text (the Chinese outro rambled for 12.9s on
# seed 49 and read cleanly in 3.1-5.5s on all ten neighbouring seeds), so a
# re-roll fixes it. Nothing downstream would notice: the runner sizes the
# cue's wall-clock to the measured audio, so the babble would stretch the cut
# and play out over the video.
#
# Detection is relative, never absolute: speech rate is language-dependent
# (~13 chars/s of English, ~6 of Chinese), so each storyboard is scored
# against its OWN median seconds-per-character. A ramble sits ~5x over that
# median while honest variation stays under ~1.4x, which is a wide gap to put
# a threshold in. Needs enough cues for the median to mean anything; the
# median of 4+ is unmoved by a single outlier, and every storyboard has 4+.
RAMBLE_TOLERANCE = 1.8
RAMBLE_MIN_CUES = 4
# Re-roll with CONSECUTIVE seeds (+1, +2, …), never with a round stride.
# Rambling turned out to track the seed modulo a power of ten rather than the
# seed itself: on the Chinese outro, seeds ≡ 49 (mod 1000) rambled 7 times out
# of 8 (49, 1049, 2049, 3049, 4049, 6049, 7049) while the four seeds either
# side of 49 were clean 4 times out of 4. A +1000 stride therefore re-rolls
# straight back into the same failure; +1 walks out of it. Colliding with a
# neighbouring cue's seed is harmless: different text, different audio, and
# voice identity comes from the predefined voice rather than the seed.
RAMBLE_RETRY_SEEDS = 4
# Two checkpoints: the design model mints the reference clip in the desired
# persona; the clone model conditions every cue on that reference's x-vector.
# Neither CustomVoice nor VoiceDesign support generate_voice_clone. Only the
# Base checkpoint does.
DEFAULT_DESIGN_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
DEFAULT_CLONE_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
@dataclass(frozen=True)
class VoiceSettings:
"""The storyboard's ``voice`` block, resolved against defaults."""
# Fixed reference utterance used to anchor the speaker timbre. The reference
# is generated once per (model, instruct, sampling, seed) tuple and reused
# for every cue, so all narration shares the same x-vector. Two short
# sentences exercise enough phonemes for a stable embedding without bloating
# generation time.
REFERENCE_TEXT = (
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video."
)
voice: str
language: str
temperature: float
exaggeration: float
cfg_weight: float
speed_factor: float
seed: int
@property
def repo_id(self) -> str:
"""Checkpoint that can actually speak this language."""
return ENGLISH_REPO_ID if self.language == "en" else MULTILINGUAL_REPO_ID
def _safe_load_json(path: Path) -> object | None:
try:
return json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError):
return None
def fingerprint(self) -> dict:
"""Settings that invalidate cached cue WAVs when any of them changes.
def _file_sha256(path: Path) -> str:
"""Content hash of a file, used to pin cached cues to a reference WAV.
The cue cache keys off the *settings* that produced the reference
(instruct/seed/…), but a re-mint of VoiceDesign (or render.sh copying a
different storyboard's reference into this audio dir) can swap the actual
reference waveform out from under those settings. Cloning some cues from
reference A and others from reference B yields two audibly different
speakers in one video. Hashing the bytes of the reference that was
actually used closes that gap: any change to the reference invalidates
every cue, so all cues in a render share one timbre.
"""
h = hashlib.sha256()
with path.open("rb") as fh:
for chunk in iter(lambda: fh.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
Stored at the top of index.json and compared wholesale, so adding a
field here is enough to make it cache-invalidating.
"""
return {
"voice": self.voice,
"language": self.language,
"model": self.repo_id,
"temperature": self.temperature,
"exaggeration": self.exaggeration,
"cfgWeight": self.cfg_weight,
"speedFactor": self.speed_factor,
"seed": self.seed,
}
def parse_args() -> argparse.Namespace:
@ -98,121 +147,221 @@ def parse_args() -> argparse.Namespace:
help="Root output directory; per-storyboard files live in <root>/<storyboard>/.",
)
parser.add_argument(
"--design-model",
default=os.environ.get("TTS_DESIGN_MODEL", DEFAULT_DESIGN_MODEL),
help="Checkpoint used to mint the voice reference (VoiceDesign by default).",
)
parser.add_argument(
"--clone-model",
default=os.environ.get("TTS_CLONE_MODEL", DEFAULT_CLONE_MODEL),
help="Checkpoint used to clone the cue audio from the reference (Base by default).",
)
parser.add_argument(
"--reference-audio",
type=Path,
default=(Path(os.environ["TTS_REFERENCE_AUDIO"]) if os.environ.get("TTS_REFERENCE_AUDIO") else None),
help="Path to an existing reference WAV. If set, skip VoiceDesign and clone from this.",
)
parser.add_argument(
"--reference-text",
default=os.environ.get("TTS_REFERENCE_TEXT"),
help="Transcript of --reference-audio. Required if --reference-audio is set.",
)
parser.add_argument(
"--device",
default=os.environ.get("TTS_DEVICE", "cuda:0"),
"--server",
default=None,
help=f"Chatterbox TTS server base URL (default {DEFAULT_SERVER}, or $CHATTERBOX_URL).",
)
return parser.parse_args()
def load_model(model_id: str, device: str) -> Qwen3TTSModel:
dtype = torch.bfloat16 if device.startswith("cuda") else torch.float32
print(f"[synth] loading {model_id} on {device} ({dtype})", flush=True)
return Qwen3TTSModel.from_pretrained(model_id, device_map=device, dtype=dtype)
def _post(url: str, payload: dict, timeout: float) -> bytes:
body = json.dumps(payload).encode()
request = urllib.request.Request(
url, data=body, headers={"Content-Type": "application/json"}
)
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.read()
def cached_index_matches(
index_path: Path,
cues: list[dict],
instruct: str,
language: str,
reference_text: str,
design_model: str,
clone_model: str,
reference_audio: str,
reference_hash: str,
seed: int,
temperature: float,
top_p: float,
) -> bool:
"""Return True iff index_path's cue list lines up with `cues` 1:1.
def _get_json(url: str, timeout: float) -> dict:
with urllib.request.urlopen(url, timeout=timeout) as response:
return json.loads(response.read())
Compared fields: ``cueIndex``, ``text``, ``gapBeforeMs`` plus the synth
settings (``instruct``, ``language``, reference text, models, the hash of
the reference WAV the cues were cloned from, ``seed``, ``temperature``,
``top_p``).
All cue WAV files must also exist on disk. Mismatched length, reordered
cues, a swapped reference waveform, or a missing WAV invalidate the cache.
"""
if not index_path.exists():
return False
def _describe_http_error(error: urllib.error.HTTPError) -> str:
"""Surface the server's `detail` message instead of a bare status code."""
try:
cached = json.loads(index_path.read_text())
except json.JSONDecodeError:
return False
if cached.get("instruct") != instruct or cached.get("language") != language:
return False
if cached.get("referenceText") != reference_text:
return False
if cached.get("designModel") != design_model or cached.get("cloneModel") != clone_model:
return False
if cached.get("referenceAudio", "") != reference_audio:
return False
if cached.get("referenceHash", "") != reference_hash:
return False
if int(cached.get("seed", -1)) != seed:
return False
if float(cached.get("temperature", -1)) != temperature:
return False
if float(cached.get("topP", -1)) != top_p:
return False
cached_items = cached.get("items", [])
if len(cached_items) != len(cues):
return False
for live, prev in zip(cues, cached_items):
if int(live["cueIndex"]) != int(prev.get("cueIndex", -1)):
return False
if live["text"].strip() != str(prev.get("text", "")).strip():
return False
if int(live.get("gapBeforeMs", 0)) != int(prev.get("gapBeforeMs", -1)):
return False
wav = prev.get("wav")
if not wav or not (index_path.parent / wav).exists():
return False
return True
return str(json.loads(error.read()).get("detail", error.reason))
except (json.JSONDecodeError, OSError, AttributeError):
return str(error.reason)
def load_reusable_items(
index_path: Path,
cues: list[dict],
instruct: str,
language: str,
reference_text: str,
design_model: str,
clone_model: str,
reference_audio: str,
reference_hash: str,
seed: int,
temperature: float,
top_p: float,
) -> dict[int, dict]:
"""Return cue-indexed cached items that match the current synth settings.
def model_type(server: str) -> str:
info = _get_json(f"{server}/api/model-info", INFO_TIMEOUT_S)
if not info.get("loaded"):
raise SystemExit(f"[synth] {server} reports no model loaded")
return str(info.get("type", ""))
Unlike ``cached_index_matches`` this accepts a partial index, so a long
CPU synthesis run can be resumed cue-by-cue after an interruption. The
reference-hash gate is what stops a re-mint from leaving some cues cloned
from the previous reference (a second voice) while only the edited cues
regenerate from the new one.
def ensure_model(server: str, repo_id: str) -> None:
"""Make the server serve `repo_id`, hot-swapping only if it is on the other.
Checking first matters: synth.py runs once per storyboard, so a set that
mixes English and localized cuts would otherwise pay the swap on every
single one.
"""
want_type = MODEL_TYPE_BY_REPO_ID[repo_id]
current = model_type(server)
if current == want_type:
print(f"[synth] server already on the {current} checkpoint", flush=True)
return
print(
f"[synth] hot-swapping checkpoint {current} -> {want_type} (~20s)",
flush=True,
)
try:
_post(f"{server}/save_settings", {"model": {"repo_id": repo_id}}, INFO_TIMEOUT_S)
_post(f"{server}/restart_server", {}, SWAP_TIMEOUT_S)
except urllib.error.HTTPError as error:
raise SystemExit(f"[synth] checkpoint swap failed: {_describe_http_error(error)}") from error
except urllib.error.URLError as error:
raise SystemExit(f"[synth] checkpoint swap failed: {error.reason}") from error
# Trust nothing: a swap that reports success but leaves the old checkpoint
# resident would synthesize German with the English frontend and no error.
settled = model_type(server)
if settled != want_type:
raise SystemExit(
f"[synth] server is on the {settled} checkpoint after requesting {want_type}"
)
print(f"[synth] checkpoint now {settled}", flush=True)
def generate_cue(server: str, text: str, voice: VoiceSettings, seed: int) -> bytes:
"""POST one cue to /tts and return the WAV bytes.
WAV, never MP3: the server hardcodes a bitrate-less MP3 export that lands
around 32 kbps, which is audible as phone-quality speech in the final mux.
"""
payload = {
"text": text,
"voice_mode": "predefined",
"predefined_voice_id": voice.voice,
"output_format": "wav",
"language": voice.language,
"temperature": voice.temperature,
"exaggeration": voice.exaggeration,
"cfg_weight": voice.cfg_weight,
"speed_factor": voice.speed_factor,
"seed": seed,
"split_text": False,
}
last_error: Exception | None = None
for attempt in range(1, GENERATE_ATTEMPTS + 1):
try:
return _post(f"{server}/tts", payload, GENERATE_TIMEOUT_S)
except urllib.error.HTTPError as error:
# 4xx is our payload being wrong (bad voice name, bad language);
# retrying cannot fix it, so fail loudly on the first one.
detail = _describe_http_error(error)
if error.code < 500:
raise SystemExit(f"[synth] /tts rejected cue: {detail}") from error
last_error = error
except (urllib.error.URLError, TimeoutError, OSError) as error:
last_error = error
if attempt < GENERATE_ATTEMPTS:
print(
f"[synth] /tts attempt {attempt}/{GENERATE_ATTEMPTS} failed ({last_error}); retrying",
flush=True,
)
time.sleep(RETRY_BACKOFF_S * attempt)
raise SystemExit(f"[synth] /tts failed after {GENERATE_ATTEMPTS} attempts: {last_error}")
def wav_stats(data: bytes) -> tuple[int, int]:
"""(sampleRate, durationMs) of a WAV, read straight from its header."""
with wave.open(io.BytesIO(data), "rb") as handle:
frame_rate = handle.getframerate()
frames = handle.getnframes()
if not frame_rate or not frames:
raise SystemExit("[synth] server returned a WAV with no audio")
return frame_rate, int(round(frames * 1000 / frame_rate))
def secs_per_char(text: str, duration_ms: int) -> float:
"""Speech rate, inverted so a rambling cue scores HIGH. See RAMBLE_TOLERANCE."""
return (duration_ms / 1000) / max(len(text.strip()), 1)
def ramble_baseline(items: list[dict]) -> float | None:
"""The storyboard's own median seconds-per-char, or None if uncalibratable."""
if len(items) < RAMBLE_MIN_CUES:
return None
baseline = statistics.median(secs_per_char(it["text"], it["durationMs"]) for it in items)
return baseline if baseline > 0 else None
def rambling_cues(items: list[dict]) -> list[dict]:
"""Cues running far enough past the storyboard's pace to look hallucinated."""
baseline = ramble_baseline(items)
if baseline is None:
return []
return [
it
for it in items
if secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE
]
def repair_rambling_cues(
server: str,
audio_dir: Path,
voice: VoiceSettings,
items: list[dict],
) -> int:
"""Re-roll the seed on any cue that runs far past the storyboard's pace.
Mutates `items` in place and returns how many cues were replaced. Applies
to reused cues as well as fresh ones, so a storyboard whose cache predates
this guard heals itself instead of keeping the bad take forever.
"""
baseline = ramble_baseline(items)
if baseline is None:
return 0
repaired = 0
for item in rambling_cues(items):
rate = secs_per_char(item["text"], item["durationMs"])
print(
f"[synth] cue {item['cueIndex']} runs {rate / baseline:.1f}x the storyboard's pace "
f"({item['durationMs']}ms); re-rolling the seed «{item['text']}»",
flush=True,
)
# Score candidates by distance from the storyboard's own pace: that
# rejects a truncated take as firmly as a rambling one, where
# "shortest wins" would happily keep a cue with its ending cut off.
best = None
best_error = abs(rate - baseline)
for attempt in range(1, RAMBLE_RETRY_SEEDS + 1):
seed = voice.seed + int(item["cueIndex"]) + attempt
data = generate_cue(server, item["text"].strip(), voice, seed)
sample_rate, duration_ms = wav_stats(data)
candidate_rate = secs_per_char(item["text"], duration_ms)
error = abs(candidate_rate - baseline)
if error < best_error:
best, best_error = (data, sample_rate, duration_ms, seed), error
if candidate_rate <= baseline * RAMBLE_TOLERANCE:
break
if best is None:
print(
f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it",
flush=True,
)
continue
data, sample_rate, duration_ms, seed = best
(audio_dir / item["wav"]).write_bytes(data)
item["sampleRate"] = sample_rate
item["durationMs"] = duration_ms
item["seed"] = seed
repaired += 1
settled = secs_per_char(item["text"], duration_ms) / baseline
note = "" if settled <= RAMBLE_TOLERANCE else " STILL LONG: listen before publishing"
print(
f"[synth] cue {item['cueIndex']}: {duration_ms}ms on seed {seed} "
f"({settled:.1f}x pace){note}",
flush=True,
)
return repaired
def load_reusable_items(index_path: Path, cues: list[dict], voice: VoiceSettings) -> dict[int, dict]:
"""Cue-indexed cached items that still match the script and the settings.
Partial by design, so an interrupted run resumes cue-by-cue instead of
re-synthesizing the whole storyboard.
"""
if not index_path.exists():
return {}
@ -220,21 +369,7 @@ def load_reusable_items(
cached = json.loads(index_path.read_text())
except json.JSONDecodeError:
return {}
if cached.get("instruct") != instruct or cached.get("language") != language:
return {}
if cached.get("referenceText") != reference_text:
return {}
if cached.get("designModel") != design_model or cached.get("cloneModel") != clone_model:
return {}
if cached.get("referenceAudio", "") != reference_audio:
return {}
if cached.get("referenceHash", "") != reference_hash:
return {}
if int(cached.get("seed", -1)) != seed:
return {}
if float(cached.get("temperature", -1)) != temperature:
return {}
if float(cached.get("topP", -1)) != top_p:
if {k: cached.get(k) for k in voice.fingerprint()} != voice.fingerprint():
return {}
cue_by_index = {int(c["cueIndex"]): c for c in cues}
@ -253,92 +388,43 @@ def load_reusable_items(
return reusable
def seed_everything(seed: int) -> None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def _resolve_reference(
args: argparse.Namespace,
audio_dir: Path,
instruct: str,
language: str,
reference_text: str,
seed: int,
temperature: float,
top_p: float,
) -> tuple[Path, str]:
"""Return (ref_wav_path, ref_text) for the clone step.
If --reference-audio is supplied, validate and use it directly. Otherwise
mint one via VoiceDesign (cached on disk; cache invalidates when the
persona/language/reference/sampling/seed changes). The design model is
unloaded before returning so the clone model can claim the GPU.
"""
if args.reference_audio is not None:
if not args.reference_audio.exists():
raise SystemExit(f"[synth] --reference-audio does not exist: {args.reference_audio}")
if not args.reference_text:
raise SystemExit("[synth] --reference-text is required when --reference-audio is set")
print(
f"[synth] using user-supplied reference {args.reference_audio} «{args.reference_text}»",
flush=True,
def resolve_voice(script_path: Path, block: dict) -> VoiceSettings:
voice = str(block.get("voice", "")).strip()
language = str(block.get("language", "")).strip()
if not voice or not language:
raise SystemExit(
f"[synth] {script_path} voice block needs `voice` and `language`. Re-run preflight."
)
return args.reference_audio, args.reference_text
ref_wav_path = audio_dir / "_reference.wav"
ref_meta_path = audio_dir / "_reference.meta.json"
ref_meta = {
"model": args.design_model,
"instruct": instruct,
"language": language,
"seed": seed,
"temperature": temperature,
"topP": top_p,
"text": reference_text,
}
if (
ref_wav_path.exists()
and ref_meta_path.exists()
and _safe_load_json(ref_meta_path) == ref_meta
):
print(f"[synth] reusing cached voice reference {ref_wav_path.name}", flush=True)
return ref_wav_path, reference_text
print(
f"[synth] minting voice reference via VoiceDesign: «{reference_text}»",
flush=True,
)
design_model = load_model(args.design_model, args.device)
seed_everything(seed)
ref_wavs, ref_sr = design_model.generate_voice_design(
text=[reference_text],
return VoiceSettings(
voice=voice,
language=language,
instruct=instruct,
do_sample=True,
temperature=temperature,
top_p=top_p,
temperature=float(block.get("temperature", 0.8)),
exaggeration=float(block.get("exaggeration", 0.5)),
cfg_weight=float(block.get("cfgWeight", 0.5)),
speed_factor=float(block.get("speedFactor", 1.0)),
seed=int(block.get("seed", 42)),
)
ref_audio = ref_wavs[0]
if hasattr(ref_audio, "cpu"):
ref_audio = ref_audio.cpu().float().numpy()
sf.write(str(ref_wav_path), ref_audio, ref_sr)
ref_meta_path.write_text(json.dumps(ref_meta, indent=2))
# Free the design model before loading the clone model: both are 1.7B,
# we don't want them resident at the same time.
del design_model
if torch.cuda.is_available():
torch.cuda.empty_cache()
return ref_wav_path, reference_text
def check_voice_exists(server: str, voice: str) -> None:
"""Fail before generating rather than after a storyboard of wrong audio.
/tts does reject an unknown predefined voice, but this names the mistake
and lists the alternatives instead of surfacing a server stack trace.
"""
try:
voices = _get_json(f"{server}/v1/audio/voices", INFO_TIMEOUT_S).get("voices", [])
except (urllib.error.URLError, OSError, json.JSONDecodeError):
return # Non-fatal: /tts still validates the name.
if voices and voice not in voices:
raise SystemExit(
f"[synth] unknown predefined voice {voice!r}. Available: {', '.join(sorted(voices))}"
)
def main() -> int:
args = parse_args()
server = (args.server or os.environ.get("CHATTERBOX_URL") or DEFAULT_SERVER).rstrip("/")
storyboard_dir = args.output_dir / args.storyboard
script_path = storyboard_dir / "narration-script.json"
@ -354,178 +440,103 @@ def main() -> int:
print("[synth] script has no cues; nothing to generate.", file=sys.stderr)
return 1
voice = script.get("voice")
if not voice:
print(
f"[synth] {script_path} has no `voice` block. Re-run preflight.",
file=sys.stderr,
)
block = script.get("voice")
if not block:
print(f"[synth] {script_path} has no `voice` block. Re-run preflight.", file=sys.stderr)
return 1
instruct = voice["instruct"]
language = voice["language"]
reference_text = str(voice.get("referenceText") or REFERENCE_TEXT)
temperature = float(voice.get("temperature", 0.6))
top_p = float(voice.get("topP", 0.9))
seed = int(voice.get("seed", 42))
reference_audio_cache_key = (
str(args.reference_audio.resolve()) if args.reference_audio is not None else ""
)
voice = resolve_voice(script_path, block)
audio_dir.mkdir(parents=True, exist_ok=True)
index_path = audio_dir / "index.json"
print(f"[synth] [{args.storyboard}] persona: {instruct}", flush=True)
print(f"[synth] [{args.storyboard}] server: {server}", flush=True)
print(
f"[synth] [{args.storyboard}] sampling: temperature={temperature} top_p={top_p} seed={seed} language={language}",
f"[synth] [{args.storyboard}] voice={voice.voice} language={voice.language} "
f"model={voice.repo_id}",
flush=True,
)
print(
f"[synth] [{args.storyboard}] temperature={voice.temperature} "
f"exaggeration={voice.exaggeration} cfg_weight={voice.cfg_weight} "
f"speed_factor={voice.speed_factor} seed={voice.seed}",
flush=True,
)
# Resolve the reference FIRST, before the cache check, so every downstream
# decision keys off the exact bytes the cues are cloned from. Two-stage
# generation:
# 1. VoiceDesign mints a single reference clip in the target persona
# (or the user supplies one via --reference-audio).
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions
# every cue on the reference's speaker embedding.
# Without (2), batched generation drifts timbre across cues: a persona
# prompt anchors style but not identity, so each batch item picks its
# own voice. The reference WAV is cached so subsequent runs only load
# the clone model (saves ~20s + 3.4 GB of disk download); when it is
# cached, _resolve_reference returns without loading any model, so doing
# this ahead of the skip check is cheap.
ref_wav_path, ref_text = _resolve_reference(
args, audio_dir, instruct, language, reference_text, seed, temperature, top_p
)
# Pin the cue cache to this exact reference waveform. A re-mint (or a
# reference copied in from another storyboard by render.sh) changes these
# bytes; without this gate the cue cache would keep cloned-from-the-old-
# reference WAVs alongside freshly regenerated ones: two voices in one
# video. See _file_sha256.
reference_hash = _file_sha256(ref_wav_path)
index_path = audio_dir / "index.json"
# Skip generation when the existing audio matches the script: same cue
# texts and same gapBeforeMs values in the same order, AND same synth
# settings (instruct/language/reference text + reference-WAV hash/model/
# seed/temperature/top_p). Saves ~30s of GPU time when iterating on
# activity timing without changing narration or persona.
if cached_index_matches(
index_path,
cues,
instruct,
language,
reference_text,
args.design_model,
args.clone_model,
reference_audio_cache_key,
reference_hash,
seed,
temperature,
top_p,
):
reusable = load_reusable_items(index_path, cues, voice)
# A fully cached storyboard still has to be re-checked for a rambling cue:
# durations live in the index, so spotting one costs no server call, and
# falling through to the generation path is what re-rolls its seed.
stale = rambling_cues(list(reusable.values()))
if len(reusable) == len(cues) and not stale:
print(
f"[synth] [{args.storyboard}] cached audio matches the current script: skipping generation",
f"[synth] [{args.storyboard}] cached audio matches the current script: "
"skipping generation",
flush=True,
)
return 0
texts = [c["text"].strip() for c in cues]
print(
f"[synth] cloning {len(texts)} cues from reference (x_vector_only)",
flush=True,
)
for i, t in enumerate(texts):
print(f"[synth] {i:2d}: {t}", flush=True)
# Only touch the server once we know there is something to synthesize:
# a fully cached storyboard must not trigger a 20s checkpoint swap.
check_voice_exists(server, voice.voice)
ensure_model(server, voice.repo_id)
clone_model = load_model(args.clone_model, args.device)
out_index_base = {
"storyboard": args.storyboard,
"instruct": instruct,
"language": language,
"designModel": args.design_model,
"cloneModel": args.clone_model,
"referenceAudio": reference_audio_cache_key,
"referenceText": ref_text,
"referenceHash": reference_hash,
"seed": seed,
"temperature": temperature,
"topP": top_p,
}
reusable = load_reusable_items(
index_path,
cues,
instruct,
language,
reference_text,
args.design_model,
args.clone_model,
reference_audio_cache_key,
reference_hash,
seed,
temperature,
top_p,
)
out_index_base = {"storyboard": args.storyboard, **voice.fingerprint()}
def write_index(items: list[dict]) -> None:
index_path.write_text(json.dumps({**out_index_base, "items": items}, indent=2))
items = []
for cue_index, cue in enumerate(cues):
cached_item = reusable.get(int(cue["cueIndex"]))
items: list[dict] = []
for cue in cues:
cue_index = int(cue["cueIndex"])
cached_item = reusable.get(cue_index)
if cached_item:
items.append(cached_item)
write_index(items)
print(
f"[synth] reusing {cached_item['wav']} {int(cached_item['durationMs']):>5d}ms «{cue['text']}»",
f"[synth] reusing {cached_item['wav']} "
f"{int(cached_item['durationMs']):>5d}ms «{cue['text']}»",
flush=True,
)
continue
seed_everything(seed + cue_index)
wavs, sr = clone_model.generate_voice_clone(
text=[texts[cue_index]],
language=language,
ref_audio=str(ref_wav_path),
ref_text=ref_text,
x_vector_only_mode=True,
non_streaming_mode=True,
do_sample=True,
temperature=temperature,
top_p=top_p,
)
if len(wavs) != 1:
text = cue["text"].strip()
if len(text) > SPLIT_TEXT_WARN_CHARS:
print(
f"[synth] model returned {len(wavs)} wavs for cue {cue_index}",
file=sys.stderr,
f"[synth] cue {cue_index} is {len(text)} chars; Chatterbox quality drops on "
"long single-pass text. Split the cue in the storyboard.",
flush=True,
)
return 1
audio = wavs[0]
if hasattr(audio, "cpu"):
audio = audio.cpu().float().numpy()
wav_name = f"cue_{cue['cueIndex']:03d}.wav"
wav_path = audio_dir / wav_name
sf.write(str(wav_path), audio, sr)
duration_ms = int(round(len(audio) * 1000 / sr))
# Vary the seed per cue: one seed for every cue in a storyboard makes
# consecutive lines land on near-identical intonation contours, which
# reads as robotic across a whole video. The voice identity is pinned
# by the predefined voice, not the seed, so this costs no consistency.
seed = voice.seed + cue_index
wav_bytes = generate_cue(server, text, voice, seed)
sample_rate, duration_ms = wav_stats(wav_bytes)
wav_name = f"cue_{cue_index:03d}.wav"
(audio_dir / wav_name).write_bytes(wav_bytes)
items.append(
{
"cueIndex": cue["cueIndex"],
"cueIndex": cue_index,
"text": cue["text"],
"gapBeforeMs": int(cue.get("gapBeforeMs", 0)),
"wav": wav_name,
"sampleRate": sr,
"sampleRate": sample_rate,
"durationMs": duration_ms,
"seed": seed,
}
)
write_index(items)
print(
f"[synth] wrote {wav_name} {duration_ms:>5d}ms «{cue['text']}»",
flush=True,
)
print(f"[synth] wrote {wav_name} {duration_ms:>5d}ms «{cue['text']}»", flush=True)
repair_rambling_cues(server, audio_dir, voice, items)
write_index(items)
total_ms = sum(it["gapBeforeMs"] + it["durationMs"] for it in items)
print(
f"[synth] [{args.storyboard}] {len(items)} cues, {total_ms}ms of audio (incl. gaps) -> {audio_dir}",
f"[synth] [{args.storyboard}] {len(items)} cues, {total_ms}ms of audio "
f"(incl. gaps) -> {audio_dir}",
flush=True,
)
return 0

1295
video/tts/uv.lock generated
View file

File diff suppressed because it is too large Load diff