migrate to chatterbox

This commit is contained in:
Andras Schmelczer 2026-07-14 16:59:48 +01:00
parent 716e42a57d
commit 988c01c4f7
7 changed files with 480 additions and 1786 deletions

View file

@ -17,9 +17,10 @@ import { storyboards } from './storyboard.js';
* The cue index in each manifest is the source of truth: the runner later * The cue index in each manifest is the source of truth: the runner later
* matches storyboard cues to measured durations by index. * matches storyboard cues to measured durations by index.
*/ */
// Em/en-dashes and ellipses make Qwen3-TTS produce dramatic pauses, sighs, // Em/en-dashes and ellipses read as dramatic pauses, sighs or audible breaths
// or audible breaths. The captions still render the original (unicode-rich) // (originally observed on Qwen3-TTS; kept as a cheap safeguard on Chatterbox).
// text from the storyboard; only the synth input is sanitised. // The captions still render the original (unicode-rich) text from the
// storyboard; only the synth input is sanitised.
function normalizeForTts(text: string): string { function normalizeForTts(text: string): string {
return text return text
.replace(/\s*[\u2014\u2013]\s*/g, ', ') .replace(/\s*[\u2014\u2013]\s*/g, ', ')
@ -39,16 +40,17 @@ function emitScript(storyboard: Storyboard): string {
gapBeforeMs: cue.gapBeforeMs, gapBeforeMs: cue.gapBeforeMs,
})); }));
// The voice block is consumed by tts/synth.py. See _resolve_reference and // The voice block is consumed by tts/synth.py. See VoiceSettings.fingerprint
// the cache check there for which fields invalidate cached audio. // there for which fields invalidate cached audio.
const manifest = { const manifest = {
storyboard: storyboard.name, storyboard: storyboard.name,
voice: { voice: {
instruct: storyboard.voice.instruct, voice: storyboard.voice.voice,
language: storyboard.voice.language, language: storyboard.voice.language,
referenceText: storyboard.voice.referenceText, temperature: storyboard.voice.temperature ?? 0.8,
temperature: storyboard.voice.temperature ?? 0.6, exaggeration: storyboard.voice.exaggeration ?? 0.5,
topP: storyboard.voice.topP ?? 0.9, cfgWeight: storyboard.voice.cfgWeight ?? 0.5,
speedFactor: storyboard.voice.speedFactor ?? 1,
seed: storyboard.voice.seed ?? 42, seed: storyboard.voice.seed ?? 42,
}, },
items, items,

View file

@ -6,9 +6,8 @@ import type { DashboardRecorder } from './dashboard.js';
* *
* The storyboard is a `Storyboard`: an ordered list of narration cues, each * The storyboard is a `Storyboard`: an ordered list of narration cues, each
* carrying the activities that play alongside it. Audio is generated FIRST * carrying the activities that play alongside it. Audio is generated FIRST
* (one batched Qwen call so the voice stays consistent across cues); the * (one Chatterbox call per cue); the runner then reads the measured per-cue
* runner then reads the measured per-cue durations and slots `during` * durations and slots `during` activities inside each cue's audio window.
* activities inside each cue's audio window.
* *
* Why cue-anchored: the audio drives pacing. Re-running synth produces a new * Why cue-anchored: the audio drives pacing. Re-running synth produces a new
* set of measured durations and the storyboard self-aligns: you don't have * set of measured durations and the storyboard self-aligns: you don't have
@ -288,21 +287,29 @@ export interface VideoConfig {
posterTimeS: number; posterTimeS: number;
} }
/** Qwen3-TTS voice + language settings, sent to synth.py via the narration /** Chatterbox voice + language settings, sent to synth.py via the narration
* script. Per storyboard so we can ship a British male narrator on one cut * script. Per storyboard so we can ship a British male narrator on one cut
* and a different persona on another. */ * and a different persona on another.
*
* Chatterbox has no persona prompt: the voice IS the accent and the register.
* Anything you would once have written as an `instruct` sentence has to be
* chosen here as a voice file instead. */
export interface VoiceConfig { export interface VoiceConfig {
/** VoiceDesign persona prompt (accent, register, anti-filler directives). */ /** Predefined voice filename on the Chatterbox server, INCLUDING `.wav`
instruct: string; * (e.g. "Southern-M.wav"). `GET /v1/audio/voices` lists the valid names. */
/** Qwen3-TTS language string, e.g. "English". */ voice: string;
/** ISO code, e.g. "en", "de", "zh". Anything other than "en" makes synth.py
* swap the server onto the multilingual checkpoint. */
language: string; language: string;
/** Reference utterance used when minting a generated voice for this language. */ /** Sampling temperature (default 0.8). */
referenceText?: string;
/** Sampling temperature (default 0.6). */
temperature?: number; temperature?: number;
/** Top-p nucleus sampling (default 0.9). */ /** Expressiveness; 0.5 is natural, higher is theatrical (default 0.5). */
topP?: number; exaggeration?: number;
/** Reproducibility seed (default 42). */ /** Classifier-free guidance weight (default 0.5). */
cfgWeight?: number;
/** Post-hoc playback rate (default 1). */
speedFactor?: number;
/** Reproducibility seed (default 42); synth.py offsets it per cue. */
seed?: number; seed?: number;
} }

View file

@ -38,10 +38,10 @@ type FormFactor = 'desktop' | 'mobile';
* to the homepage. The default storyboard is named `recording` so the * to the homepage. The default storyboard is named `recording` so the
* existing homepage `/video/recording.mp4` keeps working unchanged. * existing homepage `/video/recording.mp4` keeps working unchanged.
* *
* Audio is generated first (one batched Qwen call per storyboard, using * Audio is generated first (Chatterbox, per storyboard, using its own voice
* its own voice config), so each cue's actual duration is known before * config), so each cue's actual duration is known before recording. The
* recording. The runner sizes each cue's wall-time to the measured audio * runner sizes each cue's wall-time to the measured audio length, padding
* length, padding short `during` blocks with a trailing wait. * short `during` blocks with a trailing wait.
*/ */
// School features as served by live /api/features. The data pipeline moved // School features as served by live /api/features. The data pipeline moved
@ -97,9 +97,12 @@ type RecordingLocale = 'en' | 'de' | 'zh' | 'hi';
interface RecordingLocalization { interface RecordingLocalization {
name: string; name: string;
appLanguage: string; appLanguage: string;
/** ISO code for Chatterbox. Anything but 'en' routes to the multilingual
* checkpoint; see tts/synth.py. */
ttsLanguage: string; ttsLanguage: string;
voiceInstruct: string; /** Chatterbox predefined voice filename. This is the ONLY accent control we
voiceReferenceText: string; * have, so it stands in for what used to be a written persona prompt. */
voice: string;
promptText: string; promptText: string;
travelTimeLabel: string; travelTimeLabel: string;
exportButtonTitle: string; exportButtonTitle: string;
@ -134,12 +137,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
en: { en: {
name: 'recording', name: 'recording',
appLanguage: 'en', appLanguage: 'en',
ttsLanguage: 'English', ttsLanguage: 'en',
voiceInstruct: voice: 'Southern-M.wav',
'Calm and cheerful young British male narrator from the North of England with a ' +
'strong Manchester accent.',
voiceReferenceText:
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video.",
promptText: promptText:
'First home under £600k, 35 min to central London, good schools, low crime, quiet street, fast broadband', 'First home under £600k, 35 min to central London, good schools, low crime, quiet street, fast broadband',
travelTimeLabel: 'Central London', travelTimeLabel: 'Central London',
@ -169,12 +168,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
de: { de: {
name: 'recording-de', name: 'recording-de',
appLanguage: 'de', appLanguage: 'de',
ttsLanguage: 'German', ttsLanguage: 'de',
voiceInstruct: voice: 'German-M.wav',
'Calm and cheerful German male narrator with clear standard German pronunciation ' +
'and a friendly, practical delivery.',
voiceReferenceText:
'Willkommen zur Demonstration. Diese Sprecherstimme hörst du im gesamten Video.',
promptText: promptText:
'Wohnung unter £600k, 35 Min. ins Zentrum von London, gute Schulen, niedrige Kriminalität, ruhige Straßen', 'Wohnung unter £600k, 35 Min. ins Zentrum von London, gute Schulen, niedrige Kriminalität, ruhige Straßen',
travelTimeLabel: 'Zentrum von London', travelTimeLabel: 'Zentrum von London',
@ -205,11 +200,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
zh: { zh: {
name: 'recording-zh', name: 'recording-zh',
appLanguage: 'zh', appLanguage: 'zh',
ttsLanguage: 'Chinese', ttsLanguage: 'zh',
voiceInstruct: voice: 'Mandarin-M.wav',
'Calm and cheerful Mandarin Chinese male narrator with clear standard Mandarin ' +
'pronunciation and a friendly, practical delivery.',
voiceReferenceText: '欢迎观看演示。整段视频都会使用这位旁白的声音。',
promptText: '60万英镑以内的公寓35分钟到伦敦市中心学校好犯罪率低街道安静', promptText: '60万英镑以内的公寓35分钟到伦敦市中心学校好犯罪率低街道安静',
travelTimeLabel: '伦敦市中心', travelTimeLabel: '伦敦市中心',
exportButtonTitle: '导出为 Excel', exportButtonTitle: '导出为 Excel',
@ -235,12 +227,10 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
hi: { hi: {
name: 'recording-hi', name: 'recording-hi',
appLanguage: 'hi', appLanguage: 'hi',
ttsLanguage: 'English', // Hindi UI, English narration: the cues below are English on purpose.
voiceInstruct: // Indian-M.wav carries the Indian accent the old persona prompt asked for.
'Calm and cheerful Indian male narrator speaking English with a strong Indian accent ' + ttsLanguage: 'en',
'and a friendly, practical delivery.', voice: 'Indian-M.wav',
voiceReferenceText:
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video.",
promptText: 'Flat under £600k, 35 min to central London, good schools, low crime, quiet streets', promptText: 'Flat under £600k, 35 min to central London, good schools, low crime, quiet streets',
travelTimeLabel: 'Central London', travelTimeLabel: 'Central London',
exportButtonTitle: 'Excel में export करें', exportButtonTitle: 'Excel में export करें',
@ -568,11 +558,12 @@ function createRecordingStoryboard(
locale: formFactor === 'mobile' ? `${locale}-mobile` : locale, locale: formFactor === 'mobile' ? `${locale}-mobile` : locale,
video: buildVideoConfig(formFactor), video: buildVideoConfig(formFactor),
voice: { voice: {
instruct: copy.voiceInstruct, voice: copy.voice,
language: copy.ttsLanguage, language: copy.ttsLanguage,
referenceText: copy.voiceReferenceText, temperature: 0.8,
temperature: 0.6, exaggeration: 0.5,
topP: 0.9, cfgWeight: 0.5,
speedFactor: 1,
seed: 42, seed: 42,
}, },
content: { content: {
@ -715,20 +706,18 @@ const AD_BRAND = {
}; };
/** /**
* Ad voice persona. The SAME config is used across every ad so the voice * Ad voice. The SAME config is used across every ad so the timbre stays
* timbre stays consistent across the set (render.sh additionally reuses the * consistent across the set; with Chatterbox that consistency is free, since
* first minted reference WAV for all of them). * a predefined voice is a fixed server-side asset rather than a clip we mint
* per render.
*/ */
const AD_VOICE = { const AD_VOICE = {
instruct: voice: 'Southern-M.wav',
'British male creator-style narrator. Warm, confident, conversational, with a ' + language: 'en',
'slightly quick pace, like telling a friend about a great find. No salesy hype, ' + temperature: 0.8,
'no exaggeration. Short sentences, natural delivery.', exaggeration: 0.5,
language: 'English', cfgWeight: 0.5,
referenceText: speedFactor: 1,
'This is a short social video for people choosing where to live in England.',
temperature: 0.58,
topP: 0.9,
seed: 87, seed: 87,
}; };

View file

@ -3,7 +3,7 @@
Reads two manifests inside ``output/<storyboard>/``: Reads two manifests inside ``output/<storyboard>/``:
* ``audio/index.json`` (synth output): per-cue WAV filename + measured * ``audio/index.json`` (synth output): per-cue WAV filename + measured
duration. Generated BEFORE recording in one batched Qwen3-TTS call. duration. Generated BEFORE recording by ``tts/synth.py``.
* ``narration.json`` (recorder output): per-cue ``videoTimeMs`` against * ``narration.json`` (recorder output): per-cue ``videoTimeMs`` against
the trimmed video. Generated DURING recording. the trimmed video. Generated DURING recording.
@ -30,7 +30,7 @@ from pathlib import Path
# Mixing the narration WAVs raw left the muxed track at ~-24 LUFS: viewers # Mixing the narration WAVs raw left the muxed track at ~-24 LUFS: viewers
# had to crank the volume, which reads as low production quality. Target a # had to crank the volume, which reads as low production quality. Target a
# touch under the platform norm so speech-only audio never pumps. # touch under the platform norm so speech-only audio never pumps.
LOUDNORM_TARGET = "I=-15:TP=-1.5:LRA=11" LOUDNORM_TARGET = "I=-14:TP=-1.5:LRA=11"
def measure_loudness(cmd_head: list[str], filter_complex: str) -> dict[str, str] | None: def measure_loudness(cmd_head: list[str], filter_complex: str) -> dict[str, str] | None:

View file

@ -1,41 +1,15 @@
[project] [project]
name = "property-map-video-tts" name = "property-map-video-tts"
version = "0.1.0" version = "0.1.0"
description = "Qwen3-TTS narration generator for the homepage demo video." description = "Chatterbox narration generator + ffmpeg muxer for the homepage demo video."
requires-python = ">=3.12,<3.13" requires-python = ">=3.12"
dependencies = [ # Deliberately dependency-free. synth.py talks to the Chatterbox TTS server
"qwen-tts>=0.1.1", # over HTTP (urllib) and reads WAV headers with the stdlib `wave` module;
# Host driver is CUDA 12.4 (see `nvidia-smi`). torch 2.7+ dropped cu124 # mux.py shells out to ffmpeg. The previous Qwen3-TTS implementation ran the
# wheels, so we cap below that and pull the cu124 build from PyTorch's # model in-process, which dragged in torch + torchaudio + a CUDA-matched
# own index (configured below). torchaudio must match torch's CUDA build: # flash-attn wheel (~700MB of downloads, pinned to the host's driver version
# the PyPI default ships a CUDA 13 binary that fails to load # and to cp312). Generating over HTTP moved all of that onto the server.
# libcudart.so.13 on this host. dependencies = []
"torch>=2.5,<2.7",
"torchaudio>=2.5,<2.7",
"soundfile>=0.12",
"numpy>=1.26",
]
[project.optional-dependencies]
# Flash-attention prebuilt wheel matched to torch 2.6 + cu12 + cp312, old
# CXX ABI (PyTorch's cu124 wheel reports compiled_with_cxx11_abi() == False
# and only exports the old-ABI c10::Error constructor). Pinned to
# 2.7.4.post1 because 2.8.x's torch2.6/abiFALSE wheels were mislabelled:
# they ship new-ABI symbols and fail to import. Building from source needs
# nvcc which isn't on the host. Enable via `uv sync --extra gpu`; render.sh
# does this automatically when nvidia-smi reports a GPU.
gpu = [
"flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1%2Bcu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
]
[tool.uv] [tool.uv]
environments = ["sys_platform == 'linux' and python_version < '3.13'"] environments = ["sys_platform == 'linux' and python_version < '3.14'"]
[tool.uv.sources]
torch = [{ index = "pytorch-cu124" }]
torchaudio = [{ index = "pytorch-cu124" }]
[[tool.uv.index]]
name = "pytorch-cu124"
url = "https://download.pytorch.org/whl/cu124"
explicit = true

View file

@ -1,26 +1,34 @@
"""Synthesize one storyboard's narration in ONE batched Qwen3-TTS call. """Synthesize one storyboard's narration with the Chatterbox TTS server.
Reads ``output/<storyboard>/narration-script.json`` (emitted by Reads ``output/<storyboard>/narration-script.json`` (emitted by
``dist/preflight.js``) and runs ``Qwen3TTSModel.generate_voice_design`` with ``dist/preflight.js``) and POSTs one cue at a time to the Chatterbox server's
all cue texts as a single batched list: that way every cue shares the same native ``/tts`` endpoint. Per-cue WAVs and an index manifest go to
model state, which keeps prosody and timbre consistent across cues. Per-cue ``output/<storyboard>/audio/`` for the recording step (which reads measured
WAVs and an index manifest go to ``output/<storyboard>/audio/`` for the cue durations) and the mux step (which drops each WAV at its videoTime).
recording step (which reads measured cue durations) and the mux step (which
drops each WAV at its videoTime).
Voice persona, language, and sampling come from the storyboard via the Voice, language and sampling come from the storyboard via the ``voice`` block
``voice`` block of the narration script. CLI flags can still override them of the narration script.
for ad-hoc experimentation; storyboards remain the source of truth for
production runs.
We use the VoiceDesign sibling of CustomVoice because it accepts a free-form Chatterbox ships two checkpoints and the server holds exactly ONE at a time:
voice persona (British accent, narrator register, "no laughter") via the
``instruct`` parameter. CustomVoice's preset speakers are all American or
non-English, and its ``instruct`` is documented for emotion only. It
ignored accent directives and bled non-speech tokens (laughter, sighs)
between cues.
Run from the ``video/`` directory: * ``chatterbox`` (reported by the server as type ``original``) is English-only.
* ``multilingual`` covers 23 languages, German and Chinese among them.
We pick the checkpoint from ``voice.language`` and hot-swap the server when it
is on the wrong one. The swap is load-bearing, not cosmetic: the original
checkpoint happily ACCEPTS ``language: "de"`` and returns HTTP 200, but reads
the text with an English frontend, so a missed swap yields confident-sounding
rubbish instead of an error. Nothing downstream would catch that, hence
``ensure_model``.
Unlike the Qwen3-TTS pipeline this replaces, there is no voice-design or
reference-cloning stage: a Chatterbox predefined voice IS the reference, it
lives on the server, and it is byte-identical for every cue and every
storyboard. That deletes the whole minted-reference/referenceHash machinery
along with the "two voices in one video" class of bug it existed to prevent.
Run from the ``video/`` directory (the server must already be reachable;
render.sh health-checks it first):
uv run --project tts python tts/synth.py --storyboard recording uv run --project tts python tts/synth.py --storyboard recording
""" """
@ -28,60 +36,101 @@ Run from the ``video/`` directory:
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import hashlib import io
import json import json
import os import os
import random import statistics
import sys import sys
import time
import urllib.error
import urllib.request
import wave
from dataclasses import dataclass
from pathlib import Path from pathlib import Path
import numpy as np DEFAULT_SERVER = "http://host.docker.internal:8004"
import soundfile as sf
import torch # repo_id we POST to /save_settings -> the `type` /api/model-info reports back.
from qwen_tts import Qwen3TTSModel # The two vocabularies differ, so we cannot compare them directly.
ENGLISH_REPO_ID = "chatterbox"
MULTILINGUAL_REPO_ID = "multilingual"
MODEL_TYPE_BY_REPO_ID = {ENGLISH_REPO_ID: "original", MULTILINGUAL_REPO_ID: "multilingual"}
# Cues are one or two sentences (the longest across every storyboard is ~105
# chars), so we synthesize each in a single pass: server-side splitting would
# only add chunk seams inside a cue for no benefit. If narration ever grows
# past this, revisit rather than silently degrade.
SPLIT_TEXT_WARN_CHARS = 300
# A hot-swap unloads the current checkpoint, clears VRAM and loads the other
# one; it took ~22s on the dev box. Generation itself is ~3s per cue.
SWAP_TIMEOUT_S = 300
GENERATE_TIMEOUT_S = 300
INFO_TIMEOUT_S = 30
GENERATE_ATTEMPTS = 3
RETRY_BACKOFF_S = 2.0
# Chatterbox sometimes keeps going after it has finished the line: the cue's
# words all arrive, then several seconds of invented babble follow. It is a
# property of the seed, not the text (the Chinese outro rambled for 12.9s on
# seed 49 and read cleanly in 3.1-5.5s on all ten neighbouring seeds), so a
# re-roll fixes it. Nothing downstream would notice: the runner sizes the
# cue's wall-clock to the measured audio, so the babble would stretch the cut
# and play out over the video.
#
# Detection is relative, never absolute: speech rate is language-dependent
# (~13 chars/s of English, ~6 of Chinese), so each storyboard is scored
# against its OWN median seconds-per-character. A ramble sits ~5x over that
# median while honest variation stays under ~1.4x, which is a wide gap to put
# a threshold in. Needs enough cues for the median to mean anything; the
# median of 4+ is unmoved by a single outlier, and every storyboard has 4+.
RAMBLE_TOLERANCE = 1.8
RAMBLE_MIN_CUES = 4
# Re-roll with CONSECUTIVE seeds (+1, +2, …), never with a round stride.
# Rambling turned out to track the seed modulo a power of ten rather than the
# seed itself: on the Chinese outro, seeds ≡ 49 (mod 1000) rambled 7 times out
# of 8 (49, 1049, 2049, 3049, 4049, 6049, 7049) while the four seeds either
# side of 49 were clean 4 times out of 4. A +1000 stride therefore re-rolls
# straight back into the same failure; +1 walks out of it. Colliding with a
# neighbouring cue's seed is harmless: different text, different audio, and
# voice identity comes from the predefined voice rather than the seed.
RAMBLE_RETRY_SEEDS = 4
# Two checkpoints: the design model mints the reference clip in the desired @dataclass(frozen=True)
# persona; the clone model conditions every cue on that reference's x-vector. class VoiceSettings:
# Neither CustomVoice nor VoiceDesign support generate_voice_clone. Only the """The storyboard's ``voice`` block, resolved against defaults."""
# Base checkpoint does.
DEFAULT_DESIGN_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
DEFAULT_CLONE_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
# Fixed reference utterance used to anchor the speaker timbre. The reference voice: str
# is generated once per (model, instruct, sampling, seed) tuple and reused language: str
# for every cue, so all narration shares the same x-vector. Two short temperature: float
# sentences exercise enough phonemes for a stable embedding without bloating exaggeration: float
# generation time. cfg_weight: float
REFERENCE_TEXT = ( speed_factor: float
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video." seed: int
)
@property
def repo_id(self) -> str:
"""Checkpoint that can actually speak this language."""
return ENGLISH_REPO_ID if self.language == "en" else MULTILINGUAL_REPO_ID
def _safe_load_json(path: Path) -> object | None: def fingerprint(self) -> dict:
try: """Settings that invalidate cached cue WAVs when any of them changes.
return json.loads(path.read_text())
except (FileNotFoundError, json.JSONDecodeError):
return None
Stored at the top of index.json and compared wholesale, so adding a
def _file_sha256(path: Path) -> str: field here is enough to make it cache-invalidating.
"""Content hash of a file, used to pin cached cues to a reference WAV. """
return {
The cue cache keys off the *settings* that produced the reference "voice": self.voice,
(instruct/seed/), but a re-mint of VoiceDesign (or render.sh copying a "language": self.language,
different storyboard's reference into this audio dir) can swap the actual "model": self.repo_id,
reference waveform out from under those settings. Cloning some cues from "temperature": self.temperature,
reference A and others from reference B yields two audibly different "exaggeration": self.exaggeration,
speakers in one video. Hashing the bytes of the reference that was "cfgWeight": self.cfg_weight,
actually used closes that gap: any change to the reference invalidates "speedFactor": self.speed_factor,
every cue, so all cues in a render share one timbre. "seed": self.seed,
""" }
h = hashlib.sha256()
with path.open("rb") as fh:
for chunk in iter(lambda: fh.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def parse_args() -> argparse.Namespace: def parse_args() -> argparse.Namespace:
@ -98,121 +147,221 @@ def parse_args() -> argparse.Namespace:
help="Root output directory; per-storyboard files live in <root>/<storyboard>/.", help="Root output directory; per-storyboard files live in <root>/<storyboard>/.",
) )
parser.add_argument( parser.add_argument(
"--design-model", "--server",
default=os.environ.get("TTS_DESIGN_MODEL", DEFAULT_DESIGN_MODEL), default=None,
help="Checkpoint used to mint the voice reference (VoiceDesign by default).", help=f"Chatterbox TTS server base URL (default {DEFAULT_SERVER}, or $CHATTERBOX_URL).",
)
parser.add_argument(
"--clone-model",
default=os.environ.get("TTS_CLONE_MODEL", DEFAULT_CLONE_MODEL),
help="Checkpoint used to clone the cue audio from the reference (Base by default).",
)
parser.add_argument(
"--reference-audio",
type=Path,
default=(Path(os.environ["TTS_REFERENCE_AUDIO"]) if os.environ.get("TTS_REFERENCE_AUDIO") else None),
help="Path to an existing reference WAV. If set, skip VoiceDesign and clone from this.",
)
parser.add_argument(
"--reference-text",
default=os.environ.get("TTS_REFERENCE_TEXT"),
help="Transcript of --reference-audio. Required if --reference-audio is set.",
)
parser.add_argument(
"--device",
default=os.environ.get("TTS_DEVICE", "cuda:0"),
) )
return parser.parse_args() return parser.parse_args()
def load_model(model_id: str, device: str) -> Qwen3TTSModel: def _post(url: str, payload: dict, timeout: float) -> bytes:
dtype = torch.bfloat16 if device.startswith("cuda") else torch.float32 body = json.dumps(payload).encode()
print(f"[synth] loading {model_id} on {device} ({dtype})", flush=True) request = urllib.request.Request(
return Qwen3TTSModel.from_pretrained(model_id, device_map=device, dtype=dtype) url, data=body, headers={"Content-Type": "application/json"}
)
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.read()
def cached_index_matches( def _get_json(url: str, timeout: float) -> dict:
index_path: Path, with urllib.request.urlopen(url, timeout=timeout) as response:
cues: list[dict], return json.loads(response.read())
instruct: str,
language: str,
reference_text: str,
design_model: str,
clone_model: str,
reference_audio: str,
reference_hash: str,
seed: int,
temperature: float,
top_p: float,
) -> bool:
"""Return True iff index_path's cue list lines up with `cues` 1:1.
Compared fields: ``cueIndex``, ``text``, ``gapBeforeMs`` plus the synth
settings (``instruct``, ``language``, reference text, models, the hash of def _describe_http_error(error: urllib.error.HTTPError) -> str:
the reference WAV the cues were cloned from, ``seed``, ``temperature``, """Surface the server's `detail` message instead of a bare status code."""
``top_p``).
All cue WAV files must also exist on disk. Mismatched length, reordered
cues, a swapped reference waveform, or a missing WAV invalidate the cache.
"""
if not index_path.exists():
return False
try: try:
cached = json.loads(index_path.read_text()) return str(json.loads(error.read()).get("detail", error.reason))
except json.JSONDecodeError: except (json.JSONDecodeError, OSError, AttributeError):
return False return str(error.reason)
if cached.get("instruct") != instruct or cached.get("language") != language:
return False
if cached.get("referenceText") != reference_text:
return False
if cached.get("designModel") != design_model or cached.get("cloneModel") != clone_model:
return False
if cached.get("referenceAudio", "") != reference_audio:
return False
if cached.get("referenceHash", "") != reference_hash:
return False
if int(cached.get("seed", -1)) != seed:
return False
if float(cached.get("temperature", -1)) != temperature:
return False
if float(cached.get("topP", -1)) != top_p:
return False
cached_items = cached.get("items", [])
if len(cached_items) != len(cues):
return False
for live, prev in zip(cues, cached_items):
if int(live["cueIndex"]) != int(prev.get("cueIndex", -1)):
return False
if live["text"].strip() != str(prev.get("text", "")).strip():
return False
if int(live.get("gapBeforeMs", 0)) != int(prev.get("gapBeforeMs", -1)):
return False
wav = prev.get("wav")
if not wav or not (index_path.parent / wav).exists():
return False
return True
def load_reusable_items( def model_type(server: str) -> str:
index_path: Path, info = _get_json(f"{server}/api/model-info", INFO_TIMEOUT_S)
cues: list[dict], if not info.get("loaded"):
instruct: str, raise SystemExit(f"[synth] {server} reports no model loaded")
language: str, return str(info.get("type", ""))
reference_text: str,
design_model: str,
clone_model: str,
reference_audio: str,
reference_hash: str,
seed: int,
temperature: float,
top_p: float,
) -> dict[int, dict]:
"""Return cue-indexed cached items that match the current synth settings.
Unlike ``cached_index_matches`` this accepts a partial index, so a long
CPU synthesis run can be resumed cue-by-cue after an interruption. The def ensure_model(server: str, repo_id: str) -> None:
reference-hash gate is what stops a re-mint from leaving some cues cloned """Make the server serve `repo_id`, hot-swapping only if it is on the other.
from the previous reference (a second voice) while only the edited cues
regenerate from the new one. Checking first matters: synth.py runs once per storyboard, so a set that
mixes English and localized cuts would otherwise pay the swap on every
single one.
"""
want_type = MODEL_TYPE_BY_REPO_ID[repo_id]
current = model_type(server)
if current == want_type:
print(f"[synth] server already on the {current} checkpoint", flush=True)
return
print(
f"[synth] hot-swapping checkpoint {current} -> {want_type} (~20s)",
flush=True,
)
try:
_post(f"{server}/save_settings", {"model": {"repo_id": repo_id}}, INFO_TIMEOUT_S)
_post(f"{server}/restart_server", {}, SWAP_TIMEOUT_S)
except urllib.error.HTTPError as error:
raise SystemExit(f"[synth] checkpoint swap failed: {_describe_http_error(error)}") from error
except urllib.error.URLError as error:
raise SystemExit(f"[synth] checkpoint swap failed: {error.reason}") from error
# Trust nothing: a swap that reports success but leaves the old checkpoint
# resident would synthesize German with the English frontend and no error.
settled = model_type(server)
if settled != want_type:
raise SystemExit(
f"[synth] server is on the {settled} checkpoint after requesting {want_type}"
)
print(f"[synth] checkpoint now {settled}", flush=True)
def generate_cue(server: str, text: str, voice: VoiceSettings, seed: int) -> bytes:
"""POST one cue to /tts and return the WAV bytes.
WAV, never MP3: the server hardcodes a bitrate-less MP3 export that lands
around 32 kbps, which is audible as phone-quality speech in the final mux.
"""
payload = {
"text": text,
"voice_mode": "predefined",
"predefined_voice_id": voice.voice,
"output_format": "wav",
"language": voice.language,
"temperature": voice.temperature,
"exaggeration": voice.exaggeration,
"cfg_weight": voice.cfg_weight,
"speed_factor": voice.speed_factor,
"seed": seed,
"split_text": False,
}
last_error: Exception | None = None
for attempt in range(1, GENERATE_ATTEMPTS + 1):
try:
return _post(f"{server}/tts", payload, GENERATE_TIMEOUT_S)
except urllib.error.HTTPError as error:
# 4xx is our payload being wrong (bad voice name, bad language);
# retrying cannot fix it, so fail loudly on the first one.
detail = _describe_http_error(error)
if error.code < 500:
raise SystemExit(f"[synth] /tts rejected cue: {detail}") from error
last_error = error
except (urllib.error.URLError, TimeoutError, OSError) as error:
last_error = error
if attempt < GENERATE_ATTEMPTS:
print(
f"[synth] /tts attempt {attempt}/{GENERATE_ATTEMPTS} failed ({last_error}); retrying",
flush=True,
)
time.sleep(RETRY_BACKOFF_S * attempt)
raise SystemExit(f"[synth] /tts failed after {GENERATE_ATTEMPTS} attempts: {last_error}")
def wav_stats(data: bytes) -> tuple[int, int]:
"""(sampleRate, durationMs) of a WAV, read straight from its header."""
with wave.open(io.BytesIO(data), "rb") as handle:
frame_rate = handle.getframerate()
frames = handle.getnframes()
if not frame_rate or not frames:
raise SystemExit("[synth] server returned a WAV with no audio")
return frame_rate, int(round(frames * 1000 / frame_rate))
def secs_per_char(text: str, duration_ms: int) -> float:
"""Speech rate, inverted so a rambling cue scores HIGH. See RAMBLE_TOLERANCE."""
return (duration_ms / 1000) / max(len(text.strip()), 1)
def ramble_baseline(items: list[dict]) -> float | None:
"""The storyboard's own median seconds-per-char, or None if uncalibratable."""
if len(items) < RAMBLE_MIN_CUES:
return None
baseline = statistics.median(secs_per_char(it["text"], it["durationMs"]) for it in items)
return baseline if baseline > 0 else None
def rambling_cues(items: list[dict]) -> list[dict]:
"""Cues running far enough past the storyboard's pace to look hallucinated."""
baseline = ramble_baseline(items)
if baseline is None:
return []
return [
it
for it in items
if secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE
]
def repair_rambling_cues(
server: str,
audio_dir: Path,
voice: VoiceSettings,
items: list[dict],
) -> int:
"""Re-roll the seed on any cue that runs far past the storyboard's pace.
Mutates `items` in place and returns how many cues were replaced. Applies
to reused cues as well as fresh ones, so a storyboard whose cache predates
this guard heals itself instead of keeping the bad take forever.
"""
baseline = ramble_baseline(items)
if baseline is None:
return 0
repaired = 0
for item in rambling_cues(items):
rate = secs_per_char(item["text"], item["durationMs"])
print(
f"[synth] cue {item['cueIndex']} runs {rate / baseline:.1f}x the storyboard's pace "
f"({item['durationMs']}ms); re-rolling the seed «{item['text']}»",
flush=True,
)
# Score candidates by distance from the storyboard's own pace: that
# rejects a truncated take as firmly as a rambling one, where
# "shortest wins" would happily keep a cue with its ending cut off.
best = None
best_error = abs(rate - baseline)
for attempt in range(1, RAMBLE_RETRY_SEEDS + 1):
seed = voice.seed + int(item["cueIndex"]) + attempt
data = generate_cue(server, item["text"].strip(), voice, seed)
sample_rate, duration_ms = wav_stats(data)
candidate_rate = secs_per_char(item["text"], duration_ms)
error = abs(candidate_rate - baseline)
if error < best_error:
best, best_error = (data, sample_rate, duration_ms, seed), error
if candidate_rate <= baseline * RAMBLE_TOLERANCE:
break
if best is None:
print(
f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it",
flush=True,
)
continue
data, sample_rate, duration_ms, seed = best
(audio_dir / item["wav"]).write_bytes(data)
item["sampleRate"] = sample_rate
item["durationMs"] = duration_ms
item["seed"] = seed
repaired += 1
settled = secs_per_char(item["text"], duration_ms) / baseline
note = "" if settled <= RAMBLE_TOLERANCE else " STILL LONG: listen before publishing"
print(
f"[synth] cue {item['cueIndex']}: {duration_ms}ms on seed {seed} "
f"({settled:.1f}x pace){note}",
flush=True,
)
return repaired
def load_reusable_items(index_path: Path, cues: list[dict], voice: VoiceSettings) -> dict[int, dict]:
"""Cue-indexed cached items that still match the script and the settings.
Partial by design, so an interrupted run resumes cue-by-cue instead of
re-synthesizing the whole storyboard.
""" """
if not index_path.exists(): if not index_path.exists():
return {} return {}
@ -220,21 +369,7 @@ def load_reusable_items(
cached = json.loads(index_path.read_text()) cached = json.loads(index_path.read_text())
except json.JSONDecodeError: except json.JSONDecodeError:
return {} return {}
if cached.get("instruct") != instruct or cached.get("language") != language: if {k: cached.get(k) for k in voice.fingerprint()} != voice.fingerprint():
return {}
if cached.get("referenceText") != reference_text:
return {}
if cached.get("designModel") != design_model or cached.get("cloneModel") != clone_model:
return {}
if cached.get("referenceAudio", "") != reference_audio:
return {}
if cached.get("referenceHash", "") != reference_hash:
return {}
if int(cached.get("seed", -1)) != seed:
return {}
if float(cached.get("temperature", -1)) != temperature:
return {}
if float(cached.get("topP", -1)) != top_p:
return {} return {}
cue_by_index = {int(c["cueIndex"]): c for c in cues} cue_by_index = {int(c["cueIndex"]): c for c in cues}
@ -253,92 +388,43 @@ def load_reusable_items(
return reusable return reusable
def seed_everything(seed: int) -> None: def resolve_voice(script_path: Path, block: dict) -> VoiceSettings:
random.seed(seed) voice = str(block.get("voice", "")).strip()
np.random.seed(seed) language = str(block.get("language", "")).strip()
torch.manual_seed(seed) if not voice or not language:
if torch.cuda.is_available(): raise SystemExit(
torch.cuda.manual_seed_all(seed) f"[synth] {script_path} voice block needs `voice` and `language`. Re-run preflight."
def _resolve_reference(
args: argparse.Namespace,
audio_dir: Path,
instruct: str,
language: str,
reference_text: str,
seed: int,
temperature: float,
top_p: float,
) -> tuple[Path, str]:
"""Return (ref_wav_path, ref_text) for the clone step.
If --reference-audio is supplied, validate and use it directly. Otherwise
mint one via VoiceDesign (cached on disk; cache invalidates when the
persona/language/reference/sampling/seed changes). The design model is
unloaded before returning so the clone model can claim the GPU.
"""
if args.reference_audio is not None:
if not args.reference_audio.exists():
raise SystemExit(f"[synth] --reference-audio does not exist: {args.reference_audio}")
if not args.reference_text:
raise SystemExit("[synth] --reference-text is required when --reference-audio is set")
print(
f"[synth] using user-supplied reference {args.reference_audio} «{args.reference_text}»",
flush=True,
) )
return args.reference_audio, args.reference_text return VoiceSettings(
voice=voice,
ref_wav_path = audio_dir / "_reference.wav"
ref_meta_path = audio_dir / "_reference.meta.json"
ref_meta = {
"model": args.design_model,
"instruct": instruct,
"language": language,
"seed": seed,
"temperature": temperature,
"topP": top_p,
"text": reference_text,
}
if (
ref_wav_path.exists()
and ref_meta_path.exists()
and _safe_load_json(ref_meta_path) == ref_meta
):
print(f"[synth] reusing cached voice reference {ref_wav_path.name}", flush=True)
return ref_wav_path, reference_text
print(
f"[synth] minting voice reference via VoiceDesign: «{reference_text}»",
flush=True,
)
design_model = load_model(args.design_model, args.device)
seed_everything(seed)
ref_wavs, ref_sr = design_model.generate_voice_design(
text=[reference_text],
language=language, language=language,
instruct=instruct, temperature=float(block.get("temperature", 0.8)),
do_sample=True, exaggeration=float(block.get("exaggeration", 0.5)),
temperature=temperature, cfg_weight=float(block.get("cfgWeight", 0.5)),
top_p=top_p, speed_factor=float(block.get("speedFactor", 1.0)),
seed=int(block.get("seed", 42)),
) )
ref_audio = ref_wavs[0]
if hasattr(ref_audio, "cpu"):
ref_audio = ref_audio.cpu().float().numpy()
sf.write(str(ref_wav_path), ref_audio, ref_sr)
ref_meta_path.write_text(json.dumps(ref_meta, indent=2))
# Free the design model before loading the clone model: both are 1.7B,
# we don't want them resident at the same time.
del design_model
if torch.cuda.is_available():
torch.cuda.empty_cache()
return ref_wav_path, reference_text def check_voice_exists(server: str, voice: str) -> None:
"""Fail before generating rather than after a storyboard of wrong audio.
/tts does reject an unknown predefined voice, but this names the mistake
and lists the alternatives instead of surfacing a server stack trace.
"""
try:
voices = _get_json(f"{server}/v1/audio/voices", INFO_TIMEOUT_S).get("voices", [])
except (urllib.error.URLError, OSError, json.JSONDecodeError):
return # Non-fatal: /tts still validates the name.
if voices and voice not in voices:
raise SystemExit(
f"[synth] unknown predefined voice {voice!r}. Available: {', '.join(sorted(voices))}"
)
def main() -> int: def main() -> int:
args = parse_args() args = parse_args()
server = (args.server or os.environ.get("CHATTERBOX_URL") or DEFAULT_SERVER).rstrip("/")
storyboard_dir = args.output_dir / args.storyboard storyboard_dir = args.output_dir / args.storyboard
script_path = storyboard_dir / "narration-script.json" script_path = storyboard_dir / "narration-script.json"
@ -354,178 +440,103 @@ def main() -> int:
print("[synth] script has no cues; nothing to generate.", file=sys.stderr) print("[synth] script has no cues; nothing to generate.", file=sys.stderr)
return 1 return 1
voice = script.get("voice") block = script.get("voice")
if not voice: if not block:
print( print(f"[synth] {script_path} has no `voice` block. Re-run preflight.", file=sys.stderr)
f"[synth] {script_path} has no `voice` block. Re-run preflight.",
file=sys.stderr,
)
return 1 return 1
instruct = voice["instruct"] voice = resolve_voice(script_path, block)
language = voice["language"]
reference_text = str(voice.get("referenceText") or REFERENCE_TEXT)
temperature = float(voice.get("temperature", 0.6))
top_p = float(voice.get("topP", 0.9))
seed = int(voice.get("seed", 42))
reference_audio_cache_key = (
str(args.reference_audio.resolve()) if args.reference_audio is not None else ""
)
audio_dir.mkdir(parents=True, exist_ok=True) audio_dir.mkdir(parents=True, exist_ok=True)
index_path = audio_dir / "index.json"
print(f"[synth] [{args.storyboard}] persona: {instruct}", flush=True) print(f"[synth] [{args.storyboard}] server: {server}", flush=True)
print( print(
f"[synth] [{args.storyboard}] sampling: temperature={temperature} top_p={top_p} seed={seed} language={language}", f"[synth] [{args.storyboard}] voice={voice.voice} language={voice.language} "
f"model={voice.repo_id}",
flush=True,
)
print(
f"[synth] [{args.storyboard}] temperature={voice.temperature} "
f"exaggeration={voice.exaggeration} cfg_weight={voice.cfg_weight} "
f"speed_factor={voice.speed_factor} seed={voice.seed}",
flush=True, flush=True,
) )
# Resolve the reference FIRST, before the cache check, so every downstream reusable = load_reusable_items(index_path, cues, voice)
# decision keys off the exact bytes the cues are cloned from. Two-stage # A fully cached storyboard still has to be re-checked for a rambling cue:
# generation: # durations live in the index, so spotting one costs no server call, and
# 1. VoiceDesign mints a single reference clip in the target persona # falling through to the generation path is what re-rolls its seed.
# (or the user supplies one via --reference-audio). stale = rambling_cues(list(reusable.values()))
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions if len(reusable) == len(cues) and not stale:
# every cue on the reference's speaker embedding.
# Without (2), batched generation drifts timbre across cues: a persona
# prompt anchors style but not identity, so each batch item picks its
# own voice. The reference WAV is cached so subsequent runs only load
# the clone model (saves ~20s + 3.4 GB of disk download); when it is
# cached, _resolve_reference returns without loading any model, so doing
# this ahead of the skip check is cheap.
ref_wav_path, ref_text = _resolve_reference(
args, audio_dir, instruct, language, reference_text, seed, temperature, top_p
)
# Pin the cue cache to this exact reference waveform. A re-mint (or a
# reference copied in from another storyboard by render.sh) changes these
# bytes; without this gate the cue cache would keep cloned-from-the-old-
# reference WAVs alongside freshly regenerated ones: two voices in one
# video. See _file_sha256.
reference_hash = _file_sha256(ref_wav_path)
index_path = audio_dir / "index.json"
# Skip generation when the existing audio matches the script: same cue
# texts and same gapBeforeMs values in the same order, AND same synth
# settings (instruct/language/reference text + reference-WAV hash/model/
# seed/temperature/top_p). Saves ~30s of GPU time when iterating on
# activity timing without changing narration or persona.
if cached_index_matches(
index_path,
cues,
instruct,
language,
reference_text,
args.design_model,
args.clone_model,
reference_audio_cache_key,
reference_hash,
seed,
temperature,
top_p,
):
print( print(
f"[synth] [{args.storyboard}] cached audio matches the current script: skipping generation", f"[synth] [{args.storyboard}] cached audio matches the current script: "
"skipping generation",
flush=True, flush=True,
) )
return 0 return 0
texts = [c["text"].strip() for c in cues] # Only touch the server once we know there is something to synthesize:
print( # a fully cached storyboard must not trigger a 20s checkpoint swap.
f"[synth] cloning {len(texts)} cues from reference (x_vector_only)", check_voice_exists(server, voice.voice)
flush=True, ensure_model(server, voice.repo_id)
)
for i, t in enumerate(texts):
print(f"[synth] {i:2d}: {t}", flush=True)
clone_model = load_model(args.clone_model, args.device) out_index_base = {"storyboard": args.storyboard, **voice.fingerprint()}
out_index_base = {
"storyboard": args.storyboard,
"instruct": instruct,
"language": language,
"designModel": args.design_model,
"cloneModel": args.clone_model,
"referenceAudio": reference_audio_cache_key,
"referenceText": ref_text,
"referenceHash": reference_hash,
"seed": seed,
"temperature": temperature,
"topP": top_p,
}
reusable = load_reusable_items(
index_path,
cues,
instruct,
language,
reference_text,
args.design_model,
args.clone_model,
reference_audio_cache_key,
reference_hash,
seed,
temperature,
top_p,
)
def write_index(items: list[dict]) -> None: def write_index(items: list[dict]) -> None:
index_path.write_text(json.dumps({**out_index_base, "items": items}, indent=2)) index_path.write_text(json.dumps({**out_index_base, "items": items}, indent=2))
items = [] items: list[dict] = []
for cue_index, cue in enumerate(cues): for cue in cues:
cached_item = reusable.get(int(cue["cueIndex"])) cue_index = int(cue["cueIndex"])
cached_item = reusable.get(cue_index)
if cached_item: if cached_item:
items.append(cached_item) items.append(cached_item)
write_index(items) write_index(items)
print( print(
f"[synth] reusing {cached_item['wav']} {int(cached_item['durationMs']):>5d}ms «{cue['text']}»", f"[synth] reusing {cached_item['wav']} "
f"{int(cached_item['durationMs']):>5d}ms «{cue['text']}»",
flush=True, flush=True,
) )
continue continue
seed_everything(seed + cue_index) text = cue["text"].strip()
wavs, sr = clone_model.generate_voice_clone( if len(text) > SPLIT_TEXT_WARN_CHARS:
text=[texts[cue_index]],
language=language,
ref_audio=str(ref_wav_path),
ref_text=ref_text,
x_vector_only_mode=True,
non_streaming_mode=True,
do_sample=True,
temperature=temperature,
top_p=top_p,
)
if len(wavs) != 1:
print( print(
f"[synth] model returned {len(wavs)} wavs for cue {cue_index}", f"[synth] cue {cue_index} is {len(text)} chars; Chatterbox quality drops on "
file=sys.stderr, "long single-pass text. Split the cue in the storyboard.",
flush=True,
) )
return 1
audio = wavs[0] # Vary the seed per cue: one seed for every cue in a storyboard makes
if hasattr(audio, "cpu"): # consecutive lines land on near-identical intonation contours, which
audio = audio.cpu().float().numpy() # reads as robotic across a whole video. The voice identity is pinned
wav_name = f"cue_{cue['cueIndex']:03d}.wav" # by the predefined voice, not the seed, so this costs no consistency.
wav_path = audio_dir / wav_name seed = voice.seed + cue_index
sf.write(str(wav_path), audio, sr) wav_bytes = generate_cue(server, text, voice, seed)
duration_ms = int(round(len(audio) * 1000 / sr)) sample_rate, duration_ms = wav_stats(wav_bytes)
wav_name = f"cue_{cue_index:03d}.wav"
(audio_dir / wav_name).write_bytes(wav_bytes)
items.append( items.append(
{ {
"cueIndex": cue["cueIndex"], "cueIndex": cue_index,
"text": cue["text"], "text": cue["text"],
"gapBeforeMs": int(cue.get("gapBeforeMs", 0)), "gapBeforeMs": int(cue.get("gapBeforeMs", 0)),
"wav": wav_name, "wav": wav_name,
"sampleRate": sr, "sampleRate": sample_rate,
"durationMs": duration_ms, "durationMs": duration_ms,
"seed": seed,
} }
) )
write_index(items) write_index(items)
print( print(f"[synth] wrote {wav_name} {duration_ms:>5d}ms «{cue['text']}»", flush=True)
f"[synth] wrote {wav_name} {duration_ms:>5d}ms «{cue['text']}»",
flush=True, repair_rambling_cues(server, audio_dir, voice, items)
)
write_index(items) write_index(items)
total_ms = sum(it["gapBeforeMs"] + it["durationMs"] for it in items) total_ms = sum(it["gapBeforeMs"] + it["durationMs"] for it in items)
print( print(
f"[synth] [{args.storyboard}] {len(items)} cues, {total_ms}ms of audio (incl. gaps) -> {audio_dir}", f"[synth] [{args.storyboard}] {len(items)} cues, {total_ms}ms of audio "
f"(incl. gaps) -> {audio_dir}",
flush=True, flush=True,
) )
return 0 return 0

1295
video/tts/uv.lock generated

File diff suppressed because it is too large Load diff