migrate to chatterbox
This commit is contained in:
parent
716e42a57d
commit
988c01c4f7
7 changed files with 480 additions and 1786 deletions
|
|
@ -17,9 +17,10 @@ import { storyboards } from './storyboard.js';
|
|||
* The cue index in each manifest is the source of truth: the runner later
|
||||
* matches storyboard cues to measured durations by index.
|
||||
*/
|
||||
// Em/en-dashes and ellipses make Qwen3-TTS produce dramatic pauses, sighs,
|
||||
// or audible breaths. The captions still render the original (unicode-rich)
|
||||
// text from the storyboard; only the synth input is sanitised.
|
||||
// Em/en-dashes and ellipses read as dramatic pauses, sighs or audible breaths
|
||||
// (originally observed on Qwen3-TTS; kept as a cheap safeguard on Chatterbox).
|
||||
// The captions still render the original (unicode-rich) text from the
|
||||
// storyboard; only the synth input is sanitised.
|
||||
function normalizeForTts(text: string): string {
|
||||
return text
|
||||
.replace(/\s*[\u2014\u2013]\s*/g, ', ')
|
||||
|
|
@ -39,16 +40,17 @@ function emitScript(storyboard: Storyboard): string {
|
|||
gapBeforeMs: cue.gapBeforeMs,
|
||||
}));
|
||||
|
||||
// The voice block is consumed by tts/synth.py. See _resolve_reference and
|
||||
// the cache check there for which fields invalidate cached audio.
|
||||
// The voice block is consumed by tts/synth.py. See VoiceSettings.fingerprint
|
||||
// there for which fields invalidate cached audio.
|
||||
const manifest = {
|
||||
storyboard: storyboard.name,
|
||||
voice: {
|
||||
instruct: storyboard.voice.instruct,
|
||||
voice: storyboard.voice.voice,
|
||||
language: storyboard.voice.language,
|
||||
referenceText: storyboard.voice.referenceText,
|
||||
temperature: storyboard.voice.temperature ?? 0.6,
|
||||
topP: storyboard.voice.topP ?? 0.9,
|
||||
temperature: storyboard.voice.temperature ?? 0.8,
|
||||
exaggeration: storyboard.voice.exaggeration ?? 0.5,
|
||||
cfgWeight: storyboard.voice.cfgWeight ?? 0.5,
|
||||
speedFactor: storyboard.voice.speedFactor ?? 1,
|
||||
seed: storyboard.voice.seed ?? 42,
|
||||
},
|
||||
items,
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ import type { DashboardRecorder } from './dashboard.js';
|
|||
*
|
||||
* The storyboard is a `Storyboard`: an ordered list of narration cues, each
|
||||
* carrying the activities that play alongside it. Audio is generated FIRST
|
||||
* (one batched Qwen call so the voice stays consistent across cues); the
|
||||
* runner then reads the measured per-cue durations and slots `during`
|
||||
* activities inside each cue's audio window.
|
||||
* (one Chatterbox call per cue); the runner then reads the measured per-cue
|
||||
* durations and slots `during` activities inside each cue's audio window.
|
||||
*
|
||||
* Why cue-anchored: the audio drives pacing. Re-running synth produces a new
|
||||
* set of measured durations and the storyboard self-aligns: you don't have
|
||||
|
|
@ -288,21 +287,29 @@ export interface VideoConfig {
|
|||
posterTimeS: number;
|
||||
}
|
||||
|
||||
/** Qwen3-TTS voice + language settings, sent to synth.py via the narration
|
||||
/** Chatterbox voice + language settings, sent to synth.py via the narration
|
||||
* script. Per storyboard so we can ship a British male narrator on one cut
|
||||
* and a different persona on another. */
|
||||
* and a different persona on another.
|
||||
*
|
||||
* Chatterbox has no persona prompt: the voice IS the accent and the register.
|
||||
* Anything you would once have written as an `instruct` sentence has to be
|
||||
* chosen here as a voice file instead. */
|
||||
export interface VoiceConfig {
|
||||
/** VoiceDesign persona prompt (accent, register, anti-filler directives). */
|
||||
instruct: string;
|
||||
/** Qwen3-TTS language string, e.g. "English". */
|
||||
/** Predefined voice filename on the Chatterbox server, INCLUDING `.wav`
|
||||
* (e.g. "Southern-M.wav"). `GET /v1/audio/voices` lists the valid names. */
|
||||
voice: string;
|
||||
/** ISO code, e.g. "en", "de", "zh". Anything other than "en" makes synth.py
|
||||
* swap the server onto the multilingual checkpoint. */
|
||||
language: string;
|
||||
/** Reference utterance used when minting a generated voice for this language. */
|
||||
referenceText?: string;
|
||||
/** Sampling temperature (default 0.6). */
|
||||
/** Sampling temperature (default 0.8). */
|
||||
temperature?: number;
|
||||
/** Top-p nucleus sampling (default 0.9). */
|
||||
topP?: number;
|
||||
/** Reproducibility seed (default 42). */
|
||||
/** Expressiveness; 0.5 is natural, higher is theatrical (default 0.5). */
|
||||
exaggeration?: number;
|
||||
/** Classifier-free guidance weight (default 0.5). */
|
||||
cfgWeight?: number;
|
||||
/** Post-hoc playback rate (default 1). */
|
||||
speedFactor?: number;
|
||||
/** Reproducibility seed (default 42); synth.py offsets it per cue. */
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -38,10 +38,10 @@ type FormFactor = 'desktop' | 'mobile';
|
|||
* to the homepage. The default storyboard is named `recording` so the
|
||||
* existing homepage `/video/recording.mp4` keeps working unchanged.
|
||||
*
|
||||
* Audio is generated first (one batched Qwen call per storyboard, using
|
||||
* its own voice config), so each cue's actual duration is known before
|
||||
* recording. The runner sizes each cue's wall-time to the measured audio
|
||||
* length, padding short `during` blocks with a trailing wait.
|
||||
* Audio is generated first (Chatterbox, per storyboard, using its own voice
|
||||
* config), so each cue's actual duration is known before recording. The
|
||||
* runner sizes each cue's wall-time to the measured audio length, padding
|
||||
* short `during` blocks with a trailing wait.
|
||||
*/
|
||||
|
||||
// School features as served by live /api/features. The data pipeline moved
|
||||
|
|
@ -97,9 +97,12 @@ type RecordingLocale = 'en' | 'de' | 'zh' | 'hi';
|
|||
interface RecordingLocalization {
|
||||
name: string;
|
||||
appLanguage: string;
|
||||
/** ISO code for Chatterbox. Anything but 'en' routes to the multilingual
|
||||
* checkpoint; see tts/synth.py. */
|
||||
ttsLanguage: string;
|
||||
voiceInstruct: string;
|
||||
voiceReferenceText: string;
|
||||
/** Chatterbox predefined voice filename. This is the ONLY accent control we
|
||||
* have, so it stands in for what used to be a written persona prompt. */
|
||||
voice: string;
|
||||
promptText: string;
|
||||
travelTimeLabel: string;
|
||||
exportButtonTitle: string;
|
||||
|
|
@ -134,12 +137,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
en: {
|
||||
name: 'recording',
|
||||
appLanguage: 'en',
|
||||
ttsLanguage: 'English',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful young British male narrator from the North of England with a ' +
|
||||
'strong Manchester accent.',
|
||||
voiceReferenceText:
|
||||
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video.",
|
||||
ttsLanguage: 'en',
|
||||
voice: 'Southern-M.wav',
|
||||
promptText:
|
||||
'First home under £600k, 35 min to central London, good schools, low crime, quiet street, fast broadband',
|
||||
travelTimeLabel: 'Central London',
|
||||
|
|
@ -169,12 +168,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
de: {
|
||||
name: 'recording-de',
|
||||
appLanguage: 'de',
|
||||
ttsLanguage: 'German',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful German male narrator with clear standard German pronunciation ' +
|
||||
'and a friendly, practical delivery.',
|
||||
voiceReferenceText:
|
||||
'Willkommen zur Demonstration. Diese Sprecherstimme hörst du im gesamten Video.',
|
||||
ttsLanguage: 'de',
|
||||
voice: 'German-M.wav',
|
||||
promptText:
|
||||
'Wohnung unter £600k, 35 Min. ins Zentrum von London, gute Schulen, niedrige Kriminalität, ruhige Straßen',
|
||||
travelTimeLabel: 'Zentrum von London',
|
||||
|
|
@ -205,11 +200,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
zh: {
|
||||
name: 'recording-zh',
|
||||
appLanguage: 'zh',
|
||||
ttsLanguage: 'Chinese',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful Mandarin Chinese male narrator with clear standard Mandarin ' +
|
||||
'pronunciation and a friendly, practical delivery.',
|
||||
voiceReferenceText: '欢迎观看演示。整段视频都会使用这位旁白的声音。',
|
||||
ttsLanguage: 'zh',
|
||||
voice: 'Mandarin-M.wav',
|
||||
promptText: '60万英镑以内的公寓，35分钟到伦敦市中心，学校好，犯罪率低，街道安静',
|
||||
travelTimeLabel: '伦敦市中心',
|
||||
exportButtonTitle: '导出为 Excel',
|
||||
|
|
@ -235,12 +227,10 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
hi: {
|
||||
name: 'recording-hi',
|
||||
appLanguage: 'hi',
|
||||
ttsLanguage: 'English',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful Indian male narrator speaking English with a strong Indian accent ' +
|
||||
'and a friendly, practical delivery.',
|
||||
voiceReferenceText:
|
||||
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video.",
|
||||
// Hindi UI, English narration: the cues below are English on purpose.
|
||||
// Indian-M.wav carries the Indian accent the old persona prompt asked for.
|
||||
ttsLanguage: 'en',
|
||||
voice: 'Indian-M.wav',
|
||||
promptText: 'Flat under £600k, 35 min to central London, good schools, low crime, quiet streets',
|
||||
travelTimeLabel: 'Central London',
|
||||
exportButtonTitle: 'Excel में export करें',
|
||||
|
|
@ -568,11 +558,12 @@ function createRecordingStoryboard(
|
|||
locale: formFactor === 'mobile' ? `${locale}-mobile` : locale,
|
||||
video: buildVideoConfig(formFactor),
|
||||
voice: {
|
||||
instruct: copy.voiceInstruct,
|
||||
voice: copy.voice,
|
||||
language: copy.ttsLanguage,
|
||||
referenceText: copy.voiceReferenceText,
|
||||
temperature: 0.6,
|
||||
topP: 0.9,
|
||||
temperature: 0.8,
|
||||
exaggeration: 0.5,
|
||||
cfgWeight: 0.5,
|
||||
speedFactor: 1,
|
||||
seed: 42,
|
||||
},
|
||||
content: {
|
||||
|
|
@ -715,20 +706,18 @@ const AD_BRAND = {
|
|||
};
|
||||
|
||||
/**
|
||||
* Ad voice persona. The SAME config is used across every ad so the voice
|
||||
* timbre stays consistent across the set (render.sh additionally reuses the
|
||||
* first minted reference WAV for all of them).
|
||||
* Ad voice. The SAME config is used across every ad so the timbre stays
|
||||
* consistent across the set; with Chatterbox that consistency is free, since
|
||||
* a predefined voice is a fixed server-side asset rather than a clip we mint
|
||||
* per render.
|
||||
*/
|
||||
const AD_VOICE = {
|
||||
instruct:
|
||||
'British male creator-style narrator. Warm, confident, conversational, with a ' +
|
||||
'slightly quick pace, like telling a friend about a great find. No salesy hype, ' +
|
||||
'no exaggeration. Short sentences, natural delivery.',
|
||||
language: 'English',
|
||||
referenceText:
|
||||
'This is a short social video for people choosing where to live in England.',
|
||||
temperature: 0.58,
|
||||
topP: 0.9,
|
||||
voice: 'Southern-M.wav',
|
||||
language: 'en',
|
||||
temperature: 0.8,
|
||||
exaggeration: 0.5,
|
||||
cfgWeight: 0.5,
|
||||
speedFactor: 1,
|
||||
seed: 87,
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
Reads two manifests inside ``output/<storyboard>/``:
|
||||
|
||||
* ``audio/index.json`` (synth output): per-cue WAV filename + measured
|
||||
duration. Generated BEFORE recording in one batched Qwen3-TTS call.
|
||||
duration. Generated BEFORE recording by ``tts/synth.py``.
|
||||
* ``narration.json`` (recorder output): per-cue ``videoTimeMs`` against
|
||||
the trimmed video. Generated DURING recording.
|
||||
|
||||
|
|
@ -30,7 +30,7 @@ from pathlib import Path
|
|||
# Mixing the narration WAVs raw left the muxed track at ~-24 LUFS: viewers
|
||||
# had to crank the volume, which reads as low production quality. Target a
|
||||
# touch under the platform norm so speech-only audio never pumps.
|
||||
LOUDNORM_TARGET = "I=-15:TP=-1.5:LRA=11"
|
||||
LOUDNORM_TARGET = "I=-14:TP=-1.5:LRA=11"
|
||||
|
||||
|
||||
def measure_loudness(cmd_head: list[str], filter_complex: str) -> dict[str, str] | None:
|
||||
|
|
|
|||
|
|
@ -1,41 +1,15 @@
|
|||
[project]
|
||||
name = "property-map-video-tts"
|
||||
version = "0.1.0"
|
||||
description = "Qwen3-TTS narration generator for the homepage demo video."
|
||||
requires-python = ">=3.12,<3.13"
|
||||
dependencies = [
|
||||
"qwen-tts>=0.1.1",
|
||||
# Host driver is CUDA 12.4 (see `nvidia-smi`). torch 2.7+ dropped cu124
|
||||
# wheels, so we cap below that and pull the cu124 build from PyTorch's
|
||||
# own index (configured below). torchaudio must match torch's CUDA build:
|
||||
# the PyPI default ships a CUDA 13 binary that fails to load
|
||||
# libcudart.so.13 on this host.
|
||||
"torch>=2.5,<2.7",
|
||||
"torchaudio>=2.5,<2.7",
|
||||
"soundfile>=0.12",
|
||||
"numpy>=1.26",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
# Flash-attention prebuilt wheel matched to torch 2.6 + cu12 + cp312, old
|
||||
# CXX ABI (PyTorch's cu124 wheel reports compiled_with_cxx11_abi() == False
|
||||
# and only exports the old-ABI c10::Error constructor). Pinned to
|
||||
# 2.7.4.post1 because 2.8.x's torch2.6/abiFALSE wheels were mislabelled:
|
||||
# they ship new-ABI symbols and fail to import. Building from source needs
|
||||
# nvcc which isn't on the host. Enable via `uv sync --extra gpu`; render.sh
|
||||
# does this automatically when nvidia-smi reports a GPU.
|
||||
gpu = [
|
||||
"flash-attn @ https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1%2Bcu12torch2.6cxx11abiFALSE-cp312-cp312-linux_x86_64.whl ; sys_platform == 'linux' and platform_machine == 'x86_64'",
|
||||
]
|
||||
description = "Chatterbox narration generator + ffmpeg muxer for the homepage demo video."
|
||||
requires-python = ">=3.12"
|
||||
# Deliberately dependency-free. synth.py talks to the Chatterbox TTS server
|
||||
# over HTTP (urllib) and reads WAV headers with the stdlib `wave` module;
|
||||
# mux.py shells out to ffmpeg. The previous Qwen3-TTS implementation ran the
|
||||
# model in-process, which dragged in torch + torchaudio + a CUDA-matched
|
||||
# flash-attn wheel (~700MB of downloads, pinned to the host's driver version
|
||||
# and to cp312). Generating over HTTP moved all of that onto the server.
|
||||
dependencies = []
|
||||
|
||||
[tool.uv]
|
||||
environments = ["sys_platform == 'linux' and python_version < '3.13'"]
|
||||
|
||||
[tool.uv.sources]
|
||||
torch = [{ index = "pytorch-cu124" }]
|
||||
torchaudio = [{ index = "pytorch-cu124" }]
|
||||
|
||||
[[tool.uv.index]]
|
||||
name = "pytorch-cu124"
|
||||
url = "https://download.pytorch.org/whl/cu124"
|
||||
explicit = true
|
||||
environments = ["sys_platform == 'linux' and python_version < '3.14'"]
|
||||
|
|
|
|||
|
|
@ -1,26 +1,34 @@
|
|||
"""Synthesize one storyboard's narration in ONE batched Qwen3-TTS call.
|
||||
"""Synthesize one storyboard's narration with the Chatterbox TTS server.
|
||||
|
||||
Reads ``output/<storyboard>/narration-script.json`` (emitted by
|
||||
``dist/preflight.js``) and runs ``Qwen3TTSModel.generate_voice_design`` with
|
||||
all cue texts as a single batched list: that way every cue shares the same
|
||||
model state, which keeps prosody and timbre consistent across cues. Per-cue
|
||||
WAVs and an index manifest go to ``output/<storyboard>/audio/`` for the
|
||||
recording step (which reads measured cue durations) and the mux step (which
|
||||
drops each WAV at its videoTime).
|
||||
``dist/preflight.js``) and POSTs one cue at a time to the Chatterbox server's
|
||||
native ``/tts`` endpoint. Per-cue WAVs and an index manifest go to
|
||||
``output/<storyboard>/audio/`` for the recording step (which reads measured
|
||||
cue durations) and the mux step (which drops each WAV at its videoTime).
|
||||
|
||||
Voice persona, language, and sampling come from the storyboard via the
|
||||
``voice`` block of the narration script. CLI flags can still override them
|
||||
for ad-hoc experimentation; storyboards remain the source of truth for
|
||||
production runs.
|
||||
Voice, language and sampling come from the storyboard via the ``voice`` block
|
||||
of the narration script.
|
||||
|
||||
We use the VoiceDesign sibling of CustomVoice because it accepts a free-form
|
||||
voice persona (British accent, narrator register, "no laughter") via the
|
||||
``instruct`` parameter. CustomVoice's preset speakers are all American or
|
||||
non-English, and its ``instruct`` is documented for emotion only. It
|
||||
ignored accent directives and bled non-speech tokens (laughter, sighs)
|
||||
between cues.
|
||||
Chatterbox ships two checkpoints and the server holds exactly ONE at a time:
|
||||
|
||||
Run from the ``video/`` directory:
|
||||
* ``chatterbox`` (reported by the server as type ``original``) is English-only.
|
||||
* ``multilingual`` covers 23 languages, German and Chinese among them.
|
||||
|
||||
We pick the checkpoint from ``voice.language`` and hot-swap the server when it
|
||||
is on the wrong one. The swap is load-bearing, not cosmetic: the original
|
||||
checkpoint happily ACCEPTS ``language: "de"`` and returns HTTP 200, but reads
|
||||
the text with an English frontend, so a missed swap yields confident-sounding
|
||||
rubbish instead of an error. Nothing downstream would catch that, hence
|
||||
``ensure_model``.
|
||||
|
||||
Unlike the Qwen3-TTS pipeline this replaces, there is no voice-design or
|
||||
reference-cloning stage: a Chatterbox predefined voice IS the reference, it
|
||||
lives on the server, and it is byte-identical for every cue and every
|
||||
storyboard. That deletes the whole minted-reference/referenceHash machinery
|
||||
along with the "two voices in one video" class of bug it existed to prevent.
|
||||
|
||||
Run from the ``video/`` directory (the server must already be reachable;
|
||||
render.sh health-checks it first):
|
||||
|
||||
uv run --project tts python tts/synth.py --storyboard recording
|
||||
"""
|
||||
|
|
@ -28,60 +36,101 @@ Run from the ``video/`` directory:
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import wave
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from qwen_tts import Qwen3TTSModel
|
||||
DEFAULT_SERVER = "http://host.docker.internal:8004"
|
||||
|
||||
# repo_id we POST to /save_settings -> the `type` /api/model-info reports back.
|
||||
# The two vocabularies differ, so we cannot compare them directly.
|
||||
ENGLISH_REPO_ID = "chatterbox"
|
||||
MULTILINGUAL_REPO_ID = "multilingual"
|
||||
MODEL_TYPE_BY_REPO_ID = {ENGLISH_REPO_ID: "original", MULTILINGUAL_REPO_ID: "multilingual"}
|
||||
|
||||
# Cues are one or two sentences (the longest across every storyboard is ~105
|
||||
# chars), so we synthesize each in a single pass: server-side splitting would
|
||||
# only add chunk seams inside a cue for no benefit. If narration ever grows
|
||||
# past this, revisit rather than silently degrade.
|
||||
SPLIT_TEXT_WARN_CHARS = 300
|
||||
|
||||
# A hot-swap unloads the current checkpoint, clears VRAM and loads the other
|
||||
# one; it took ~22s on the dev box. Generation itself is ~3s per cue.
|
||||
SWAP_TIMEOUT_S = 300
|
||||
GENERATE_TIMEOUT_S = 300
|
||||
INFO_TIMEOUT_S = 30
|
||||
|
||||
GENERATE_ATTEMPTS = 3
|
||||
RETRY_BACKOFF_S = 2.0
|
||||
|
||||
# Chatterbox sometimes keeps going after it has finished the line: the cue's
|
||||
# words all arrive, then several seconds of invented babble follow. It is a
|
||||
# property of the seed, not the text (the Chinese outro rambled for 12.9s on
|
||||
# seed 49 and read cleanly in 3.1-5.5s on all ten neighbouring seeds), so a
|
||||
# re-roll fixes it. Nothing downstream would notice: the runner sizes the
|
||||
# cue's wall-clock to the measured audio, so the babble would stretch the cut
|
||||
# and play out over the video.
|
||||
#
|
||||
# Detection is relative, never absolute: speech rate is language-dependent
|
||||
# (~13 chars/s of English, ~6 of Chinese), so each storyboard is scored
|
||||
# against its OWN median seconds-per-character. A ramble sits ~5x over that
|
||||
# median while honest variation stays under ~1.4x, which is a wide gap to put
|
||||
# a threshold in. Needs enough cues for the median to mean anything; the
|
||||
# median of 4+ is unmoved by a single outlier, and every storyboard has 4+.
|
||||
RAMBLE_TOLERANCE = 1.8
|
||||
RAMBLE_MIN_CUES = 4
|
||||
# Re-roll with CONSECUTIVE seeds (+1, +2, …), never with a round stride.
|
||||
# Rambling turned out to track the seed modulo a power of ten rather than the
|
||||
# seed itself: on the Chinese outro, seeds ≡ 49 (mod 1000) rambled 7 times out
|
||||
# of 8 (49, 1049, 2049, 3049, 4049, 6049, 7049) while the four seeds either
|
||||
# side of 49 were clean 4 times out of 4. A +1000 stride therefore re-rolls
|
||||
# straight back into the same failure; +1 walks out of it. Colliding with a
|
||||
# neighbouring cue's seed is harmless: different text, different audio, and
|
||||
# voice identity comes from the predefined voice rather than the seed.
|
||||
RAMBLE_RETRY_SEEDS = 4
|
||||
|
||||
|
||||
# Two checkpoints: the design model mints the reference clip in the desired
|
||||
# persona; the clone model conditions every cue on that reference's x-vector.
|
||||
# Neither CustomVoice nor VoiceDesign support generate_voice_clone. Only the
|
||||
# Base checkpoint does.
|
||||
DEFAULT_DESIGN_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
||||
DEFAULT_CLONE_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
@dataclass(frozen=True)
|
||||
class VoiceSettings:
|
||||
"""The storyboard's ``voice`` block, resolved against defaults."""
|
||||
|
||||
# Fixed reference utterance used to anchor the speaker timbre. The reference
|
||||
# is generated once per (model, instruct, sampling, seed) tuple and reused
|
||||
# for every cue, so all narration shares the same x-vector. Two short
|
||||
# sentences exercise enough phonemes for a stable embedding without bloating
|
||||
# generation time.
|
||||
REFERENCE_TEXT = (
|
||||
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video."
|
||||
)
|
||||
voice: str
|
||||
language: str
|
||||
temperature: float
|
||||
exaggeration: float
|
||||
cfg_weight: float
|
||||
speed_factor: float
|
||||
seed: int
|
||||
|
||||
@property
|
||||
def repo_id(self) -> str:
|
||||
"""Checkpoint that can actually speak this language."""
|
||||
return ENGLISH_REPO_ID if self.language == "en" else MULTILINGUAL_REPO_ID
|
||||
|
||||
def _safe_load_json(path: Path) -> object | None:
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return None
|
||||
def fingerprint(self) -> dict:
|
||||
"""Settings that invalidate cached cue WAVs when any of them changes.
|
||||
|
||||
|
||||
def _file_sha256(path: Path) -> str:
|
||||
"""Content hash of a file, used to pin cached cues to a reference WAV.
|
||||
|
||||
The cue cache keys off the *settings* that produced the reference
|
||||
(instruct/seed/…), but a re-mint of VoiceDesign (or render.sh copying a
|
||||
different storyboard's reference into this audio dir) can swap the actual
|
||||
reference waveform out from under those settings. Cloning some cues from
|
||||
reference A and others from reference B yields two audibly different
|
||||
speakers in one video. Hashing the bytes of the reference that was
|
||||
actually used closes that gap: any change to the reference invalidates
|
||||
every cue, so all cues in a render share one timbre.
|
||||
"""
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
Stored at the top of index.json and compared wholesale, so adding a
|
||||
field here is enough to make it cache-invalidating.
|
||||
"""
|
||||
return {
|
||||
"voice": self.voice,
|
||||
"language": self.language,
|
||||
"model": self.repo_id,
|
||||
"temperature": self.temperature,
|
||||
"exaggeration": self.exaggeration,
|
||||
"cfgWeight": self.cfg_weight,
|
||||
"speedFactor": self.speed_factor,
|
||||
"seed": self.seed,
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
|
|
@ -98,121 +147,221 @@ def parse_args() -> argparse.Namespace:
|
|||
help="Root output directory; per-storyboard files live in <root>/<storyboard>/.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--design-model",
|
||||
default=os.environ.get("TTS_DESIGN_MODEL", DEFAULT_DESIGN_MODEL),
|
||||
help="Checkpoint used to mint the voice reference (VoiceDesign by default).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clone-model",
|
||||
default=os.environ.get("TTS_CLONE_MODEL", DEFAULT_CLONE_MODEL),
|
||||
help="Checkpoint used to clone the cue audio from the reference (Base by default).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reference-audio",
|
||||
type=Path,
|
||||
default=(Path(os.environ["TTS_REFERENCE_AUDIO"]) if os.environ.get("TTS_REFERENCE_AUDIO") else None),
|
||||
help="Path to an existing reference WAV. If set, skip VoiceDesign and clone from this.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--reference-text",
|
||||
default=os.environ.get("TTS_REFERENCE_TEXT"),
|
||||
help="Transcript of --reference-audio. Required if --reference-audio is set.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
default=os.environ.get("TTS_DEVICE", "cuda:0"),
|
||||
"--server",
|
||||
default=None,
|
||||
help=f"Chatterbox TTS server base URL (default {DEFAULT_SERVER}, or $CHATTERBOX_URL).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_model(model_id: str, device: str) -> Qwen3TTSModel:
|
||||
dtype = torch.bfloat16 if device.startswith("cuda") else torch.float32
|
||||
print(f"[synth] loading {model_id} on {device} ({dtype})", flush=True)
|
||||
return Qwen3TTSModel.from_pretrained(model_id, device_map=device, dtype=dtype)
|
||||
def _post(url: str, payload: dict, timeout: float) -> bytes:
|
||||
body = json.dumps(payload).encode()
|
||||
request = urllib.request.Request(
|
||||
url, data=body, headers={"Content-Type": "application/json"}
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
return response.read()
|
||||
|
||||
|
||||
def cached_index_matches(
|
||||
index_path: Path,
|
||||
cues: list[dict],
|
||||
instruct: str,
|
||||
language: str,
|
||||
reference_text: str,
|
||||
design_model: str,
|
||||
clone_model: str,
|
||||
reference_audio: str,
|
||||
reference_hash: str,
|
||||
seed: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
) -> bool:
|
||||
"""Return True iff index_path's cue list lines up with `cues` 1:1.
|
||||
def _get_json(url: str, timeout: float) -> dict:
|
||||
with urllib.request.urlopen(url, timeout=timeout) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
Compared fields: ``cueIndex``, ``text``, ``gapBeforeMs`` plus the synth
|
||||
settings (``instruct``, ``language``, reference text, models, the hash of
|
||||
the reference WAV the cues were cloned from, ``seed``, ``temperature``,
|
||||
``top_p``).
|
||||
All cue WAV files must also exist on disk. Mismatched length, reordered
|
||||
cues, a swapped reference waveform, or a missing WAV invalidate the cache.
|
||||
"""
|
||||
if not index_path.exists():
|
||||
return False
|
||||
|
||||
def _describe_http_error(error: urllib.error.HTTPError) -> str:
|
||||
"""Surface the server's `detail` message instead of a bare status code."""
|
||||
try:
|
||||
cached = json.loads(index_path.read_text())
|
||||
except json.JSONDecodeError:
|
||||
return False
|
||||
if cached.get("instruct") != instruct or cached.get("language") != language:
|
||||
return False
|
||||
if cached.get("referenceText") != reference_text:
|
||||
return False
|
||||
if cached.get("designModel") != design_model or cached.get("cloneModel") != clone_model:
|
||||
return False
|
||||
if cached.get("referenceAudio", "") != reference_audio:
|
||||
return False
|
||||
if cached.get("referenceHash", "") != reference_hash:
|
||||
return False
|
||||
if int(cached.get("seed", -1)) != seed:
|
||||
return False
|
||||
if float(cached.get("temperature", -1)) != temperature:
|
||||
return False
|
||||
if float(cached.get("topP", -1)) != top_p:
|
||||
return False
|
||||
cached_items = cached.get("items", [])
|
||||
if len(cached_items) != len(cues):
|
||||
return False
|
||||
for live, prev in zip(cues, cached_items):
|
||||
if int(live["cueIndex"]) != int(prev.get("cueIndex", -1)):
|
||||
return False
|
||||
if live["text"].strip() != str(prev.get("text", "")).strip():
|
||||
return False
|
||||
if int(live.get("gapBeforeMs", 0)) != int(prev.get("gapBeforeMs", -1)):
|
||||
return False
|
||||
wav = prev.get("wav")
|
||||
if not wav or not (index_path.parent / wav).exists():
|
||||
return False
|
||||
return True
|
||||
return str(json.loads(error.read()).get("detail", error.reason))
|
||||
except (json.JSONDecodeError, OSError, AttributeError):
|
||||
return str(error.reason)
|
||||
|
||||
|
||||
def load_reusable_items(
|
||||
index_path: Path,
|
||||
cues: list[dict],
|
||||
instruct: str,
|
||||
language: str,
|
||||
reference_text: str,
|
||||
design_model: str,
|
||||
clone_model: str,
|
||||
reference_audio: str,
|
||||
reference_hash: str,
|
||||
seed: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
) -> dict[int, dict]:
|
||||
"""Return cue-indexed cached items that match the current synth settings.
|
||||
def model_type(server: str) -> str:
|
||||
info = _get_json(f"{server}/api/model-info", INFO_TIMEOUT_S)
|
||||
if not info.get("loaded"):
|
||||
raise SystemExit(f"[synth] {server} reports no model loaded")
|
||||
return str(info.get("type", ""))
|
||||
|
||||
Unlike ``cached_index_matches`` this accepts a partial index, so a long
|
||||
CPU synthesis run can be resumed cue-by-cue after an interruption. The
|
||||
reference-hash gate is what stops a re-mint from leaving some cues cloned
|
||||
from the previous reference (a second voice) while only the edited cues
|
||||
regenerate from the new one.
|
||||
|
||||
def ensure_model(server: str, repo_id: str) -> None:
|
||||
"""Make the server serve `repo_id`, hot-swapping only if it is on the other.
|
||||
|
||||
Checking first matters: synth.py runs once per storyboard, so a set that
|
||||
mixes English and localized cuts would otherwise pay the swap on every
|
||||
single one.
|
||||
"""
|
||||
want_type = MODEL_TYPE_BY_REPO_ID[repo_id]
|
||||
current = model_type(server)
|
||||
if current == want_type:
|
||||
print(f"[synth] server already on the {current} checkpoint", flush=True)
|
||||
return
|
||||
|
||||
print(
|
||||
f"[synth] hot-swapping checkpoint {current} -> {want_type} (~20s)",
|
||||
flush=True,
|
||||
)
|
||||
try:
|
||||
_post(f"{server}/save_settings", {"model": {"repo_id": repo_id}}, INFO_TIMEOUT_S)
|
||||
_post(f"{server}/restart_server", {}, SWAP_TIMEOUT_S)
|
||||
except urllib.error.HTTPError as error:
|
||||
raise SystemExit(f"[synth] checkpoint swap failed: {_describe_http_error(error)}") from error
|
||||
except urllib.error.URLError as error:
|
||||
raise SystemExit(f"[synth] checkpoint swap failed: {error.reason}") from error
|
||||
|
||||
# Trust nothing: a swap that reports success but leaves the old checkpoint
|
||||
# resident would synthesize German with the English frontend and no error.
|
||||
settled = model_type(server)
|
||||
if settled != want_type:
|
||||
raise SystemExit(
|
||||
f"[synth] server is on the {settled} checkpoint after requesting {want_type}"
|
||||
)
|
||||
print(f"[synth] checkpoint now {settled}", flush=True)
|
||||
|
||||
|
||||
def generate_cue(server: str, text: str, voice: VoiceSettings, seed: int) -> bytes:
|
||||
"""POST one cue to /tts and return the WAV bytes.
|
||||
|
||||
WAV, never MP3: the server hardcodes a bitrate-less MP3 export that lands
|
||||
around 32 kbps, which is audible as phone-quality speech in the final mux.
|
||||
"""
|
||||
payload = {
|
||||
"text": text,
|
||||
"voice_mode": "predefined",
|
||||
"predefined_voice_id": voice.voice,
|
||||
"output_format": "wav",
|
||||
"language": voice.language,
|
||||
"temperature": voice.temperature,
|
||||
"exaggeration": voice.exaggeration,
|
||||
"cfg_weight": voice.cfg_weight,
|
||||
"speed_factor": voice.speed_factor,
|
||||
"seed": seed,
|
||||
"split_text": False,
|
||||
}
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, GENERATE_ATTEMPTS + 1):
|
||||
try:
|
||||
return _post(f"{server}/tts", payload, GENERATE_TIMEOUT_S)
|
||||
except urllib.error.HTTPError as error:
|
||||
# 4xx is our payload being wrong (bad voice name, bad language);
|
||||
# retrying cannot fix it, so fail loudly on the first one.
|
||||
detail = _describe_http_error(error)
|
||||
if error.code < 500:
|
||||
raise SystemExit(f"[synth] /tts rejected cue: {detail}") from error
|
||||
last_error = error
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as error:
|
||||
last_error = error
|
||||
if attempt < GENERATE_ATTEMPTS:
|
||||
print(
|
||||
f"[synth] /tts attempt {attempt}/{GENERATE_ATTEMPTS} failed ({last_error}); retrying",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(RETRY_BACKOFF_S * attempt)
|
||||
raise SystemExit(f"[synth] /tts failed after {GENERATE_ATTEMPTS} attempts: {last_error}")
|
||||
|
||||
|
||||
def wav_stats(data: bytes) -> tuple[int, int]:
|
||||
"""(sampleRate, durationMs) of a WAV, read straight from its header."""
|
||||
with wave.open(io.BytesIO(data), "rb") as handle:
|
||||
frame_rate = handle.getframerate()
|
||||
frames = handle.getnframes()
|
||||
if not frame_rate or not frames:
|
||||
raise SystemExit("[synth] server returned a WAV with no audio")
|
||||
return frame_rate, int(round(frames * 1000 / frame_rate))
|
||||
|
||||
|
||||
def secs_per_char(text: str, duration_ms: int) -> float:
|
||||
"""Speech rate, inverted so a rambling cue scores HIGH. See RAMBLE_TOLERANCE."""
|
||||
return (duration_ms / 1000) / max(len(text.strip()), 1)
|
||||
|
||||
|
||||
def ramble_baseline(items: list[dict]) -> float | None:
|
||||
"""The storyboard's own median seconds-per-char, or None if uncalibratable."""
|
||||
if len(items) < RAMBLE_MIN_CUES:
|
||||
return None
|
||||
baseline = statistics.median(secs_per_char(it["text"], it["durationMs"]) for it in items)
|
||||
return baseline if baseline > 0 else None
|
||||
|
||||
|
||||
def rambling_cues(items: list[dict]) -> list[dict]:
|
||||
"""Cues running far enough past the storyboard's pace to look hallucinated."""
|
||||
baseline = ramble_baseline(items)
|
||||
if baseline is None:
|
||||
return []
|
||||
return [
|
||||
it
|
||||
for it in items
|
||||
if secs_per_char(it["text"], it["durationMs"]) > baseline * RAMBLE_TOLERANCE
|
||||
]
|
||||
|
||||
|
||||
def repair_rambling_cues(
|
||||
server: str,
|
||||
audio_dir: Path,
|
||||
voice: VoiceSettings,
|
||||
items: list[dict],
|
||||
) -> int:
|
||||
"""Re-roll the seed on any cue that runs far past the storyboard's pace.
|
||||
|
||||
Mutates `items` in place and returns how many cues were replaced. Applies
|
||||
to reused cues as well as fresh ones, so a storyboard whose cache predates
|
||||
this guard heals itself instead of keeping the bad take forever.
|
||||
"""
|
||||
baseline = ramble_baseline(items)
|
||||
if baseline is None:
|
||||
return 0
|
||||
|
||||
repaired = 0
|
||||
for item in rambling_cues(items):
|
||||
rate = secs_per_char(item["text"], item["durationMs"])
|
||||
print(
|
||||
f"[synth] cue {item['cueIndex']} runs {rate / baseline:.1f}x the storyboard's pace "
|
||||
f"({item['durationMs']}ms); re-rolling the seed «{item['text']}»",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Score candidates by distance from the storyboard's own pace: that
|
||||
# rejects a truncated take as firmly as a rambling one, where
|
||||
# "shortest wins" would happily keep a cue with its ending cut off.
|
||||
best = None
|
||||
best_error = abs(rate - baseline)
|
||||
for attempt in range(1, RAMBLE_RETRY_SEEDS + 1):
|
||||
seed = voice.seed + int(item["cueIndex"]) + attempt
|
||||
data = generate_cue(server, item["text"].strip(), voice, seed)
|
||||
sample_rate, duration_ms = wav_stats(data)
|
||||
candidate_rate = secs_per_char(item["text"], duration_ms)
|
||||
error = abs(candidate_rate - baseline)
|
||||
if error < best_error:
|
||||
best, best_error = (data, sample_rate, duration_ms, seed), error
|
||||
if candidate_rate <= baseline * RAMBLE_TOLERANCE:
|
||||
break
|
||||
|
||||
if best is None:
|
||||
print(
|
||||
f"[synth] cue {item['cueIndex']}: no re-roll beat the original; keeping it",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
data, sample_rate, duration_ms, seed = best
|
||||
(audio_dir / item["wav"]).write_bytes(data)
|
||||
item["sampleRate"] = sample_rate
|
||||
item["durationMs"] = duration_ms
|
||||
item["seed"] = seed
|
||||
repaired += 1
|
||||
settled = secs_per_char(item["text"], duration_ms) / baseline
|
||||
note = "" if settled <= RAMBLE_TOLERANCE else " STILL LONG: listen before publishing"
|
||||
print(
|
||||
f"[synth] cue {item['cueIndex']}: {duration_ms}ms on seed {seed} "
|
||||
f"({settled:.1f}x pace){note}",
|
||||
flush=True,
|
||||
)
|
||||
return repaired
|
||||
|
||||
|
||||
def load_reusable_items(index_path: Path, cues: list[dict], voice: VoiceSettings) -> dict[int, dict]:
|
||||
"""Cue-indexed cached items that still match the script and the settings.
|
||||
|
||||
Partial by design, so an interrupted run resumes cue-by-cue instead of
|
||||
re-synthesizing the whole storyboard.
|
||||
"""
|
||||
if not index_path.exists():
|
||||
return {}
|
||||
|
|
@ -220,21 +369,7 @@ def load_reusable_items(
|
|||
cached = json.loads(index_path.read_text())
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
if cached.get("instruct") != instruct or cached.get("language") != language:
|
||||
return {}
|
||||
if cached.get("referenceText") != reference_text:
|
||||
return {}
|
||||
if cached.get("designModel") != design_model or cached.get("cloneModel") != clone_model:
|
||||
return {}
|
||||
if cached.get("referenceAudio", "") != reference_audio:
|
||||
return {}
|
||||
if cached.get("referenceHash", "") != reference_hash:
|
||||
return {}
|
||||
if int(cached.get("seed", -1)) != seed:
|
||||
return {}
|
||||
if float(cached.get("temperature", -1)) != temperature:
|
||||
return {}
|
||||
if float(cached.get("topP", -1)) != top_p:
|
||||
if {k: cached.get(k) for k in voice.fingerprint()} != voice.fingerprint():
|
||||
return {}
|
||||
|
||||
cue_by_index = {int(c["cueIndex"]): c for c in cues}
|
||||
|
|
@ -253,92 +388,43 @@ def load_reusable_items(
|
|||
return reusable
|
||||
|
||||
|
||||
def seed_everything(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
def _resolve_reference(
|
||||
args: argparse.Namespace,
|
||||
audio_dir: Path,
|
||||
instruct: str,
|
||||
language: str,
|
||||
reference_text: str,
|
||||
seed: int,
|
||||
temperature: float,
|
||||
top_p: float,
|
||||
) -> tuple[Path, str]:
|
||||
"""Return (ref_wav_path, ref_text) for the clone step.
|
||||
|
||||
If --reference-audio is supplied, validate and use it directly. Otherwise
|
||||
mint one via VoiceDesign (cached on disk; cache invalidates when the
|
||||
persona/language/reference/sampling/seed changes). The design model is
|
||||
unloaded before returning so the clone model can claim the GPU.
|
||||
"""
|
||||
if args.reference_audio is not None:
|
||||
if not args.reference_audio.exists():
|
||||
raise SystemExit(f"[synth] --reference-audio does not exist: {args.reference_audio}")
|
||||
if not args.reference_text:
|
||||
raise SystemExit("[synth] --reference-text is required when --reference-audio is set")
|
||||
print(
|
||||
f"[synth] using user-supplied reference {args.reference_audio} «{args.reference_text}»",
|
||||
flush=True,
|
||||
def resolve_voice(script_path: Path, block: dict) -> VoiceSettings:
|
||||
voice = str(block.get("voice", "")).strip()
|
||||
language = str(block.get("language", "")).strip()
|
||||
if not voice or not language:
|
||||
raise SystemExit(
|
||||
f"[synth] {script_path} voice block needs `voice` and `language`. Re-run preflight."
|
||||
)
|
||||
return args.reference_audio, args.reference_text
|
||||
|
||||
ref_wav_path = audio_dir / "_reference.wav"
|
||||
ref_meta_path = audio_dir / "_reference.meta.json"
|
||||
ref_meta = {
|
||||
"model": args.design_model,
|
||||
"instruct": instruct,
|
||||
"language": language,
|
||||
"seed": seed,
|
||||
"temperature": temperature,
|
||||
"topP": top_p,
|
||||
"text": reference_text,
|
||||
}
|
||||
if (
|
||||
ref_wav_path.exists()
|
||||
and ref_meta_path.exists()
|
||||
and _safe_load_json(ref_meta_path) == ref_meta
|
||||
):
|
||||
print(f"[synth] reusing cached voice reference {ref_wav_path.name}", flush=True)
|
||||
return ref_wav_path, reference_text
|
||||
|
||||
print(
|
||||
f"[synth] minting voice reference via VoiceDesign: «{reference_text}»",
|
||||
flush=True,
|
||||
)
|
||||
design_model = load_model(args.design_model, args.device)
|
||||
seed_everything(seed)
|
||||
ref_wavs, ref_sr = design_model.generate_voice_design(
|
||||
text=[reference_text],
|
||||
return VoiceSettings(
|
||||
voice=voice,
|
||||
language=language,
|
||||
instruct=instruct,
|
||||
do_sample=True,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
temperature=float(block.get("temperature", 0.8)),
|
||||
exaggeration=float(block.get("exaggeration", 0.5)),
|
||||
cfg_weight=float(block.get("cfgWeight", 0.5)),
|
||||
speed_factor=float(block.get("speedFactor", 1.0)),
|
||||
seed=int(block.get("seed", 42)),
|
||||
)
|
||||
ref_audio = ref_wavs[0]
|
||||
if hasattr(ref_audio, "cpu"):
|
||||
ref_audio = ref_audio.cpu().float().numpy()
|
||||
sf.write(str(ref_wav_path), ref_audio, ref_sr)
|
||||
ref_meta_path.write_text(json.dumps(ref_meta, indent=2))
|
||||
|
||||
# Free the design model before loading the clone model: both are 1.7B,
|
||||
# we don't want them resident at the same time.
|
||||
del design_model
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
return ref_wav_path, reference_text
|
||||
def check_voice_exists(server: str, voice: str) -> None:
|
||||
"""Fail before generating rather than after a storyboard of wrong audio.
|
||||
|
||||
/tts does reject an unknown predefined voice, but this names the mistake
|
||||
and lists the alternatives instead of surfacing a server stack trace.
|
||||
"""
|
||||
try:
|
||||
voices = _get_json(f"{server}/v1/audio/voices", INFO_TIMEOUT_S).get("voices", [])
|
||||
except (urllib.error.URLError, OSError, json.JSONDecodeError):
|
||||
return # Non-fatal: /tts still validates the name.
|
||||
if voices and voice not in voices:
|
||||
raise SystemExit(
|
||||
f"[synth] unknown predefined voice {voice!r}. Available: {', '.join(sorted(voices))}"
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
server = (args.server or os.environ.get("CHATTERBOX_URL") or DEFAULT_SERVER).rstrip("/")
|
||||
|
||||
storyboard_dir = args.output_dir / args.storyboard
|
||||
script_path = storyboard_dir / "narration-script.json"
|
||||
|
|
@ -354,178 +440,103 @@ def main() -> int:
|
|||
print("[synth] script has no cues; nothing to generate.", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
voice = script.get("voice")
|
||||
if not voice:
|
||||
print(
|
||||
f"[synth] {script_path} has no `voice` block. Re-run preflight.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
block = script.get("voice")
|
||||
if not block:
|
||||
print(f"[synth] {script_path} has no `voice` block. Re-run preflight.", file=sys.stderr)
|
||||
return 1
|
||||
instruct = voice["instruct"]
|
||||
language = voice["language"]
|
||||
reference_text = str(voice.get("referenceText") or REFERENCE_TEXT)
|
||||
temperature = float(voice.get("temperature", 0.6))
|
||||
top_p = float(voice.get("topP", 0.9))
|
||||
seed = int(voice.get("seed", 42))
|
||||
reference_audio_cache_key = (
|
||||
str(args.reference_audio.resolve()) if args.reference_audio is not None else ""
|
||||
)
|
||||
voice = resolve_voice(script_path, block)
|
||||
|
||||
audio_dir.mkdir(parents=True, exist_ok=True)
|
||||
index_path = audio_dir / "index.json"
|
||||
|
||||
print(f"[synth] [{args.storyboard}] persona: {instruct}", flush=True)
|
||||
print(f"[synth] [{args.storyboard}] server: {server}", flush=True)
|
||||
print(
|
||||
f"[synth] [{args.storyboard}] sampling: temperature={temperature} top_p={top_p} seed={seed} language={language}",
|
||||
f"[synth] [{args.storyboard}] voice={voice.voice} language={voice.language} "
|
||||
f"model={voice.repo_id}",
|
||||
flush=True,
|
||||
)
|
||||
print(
|
||||
f"[synth] [{args.storyboard}] temperature={voice.temperature} "
|
||||
f"exaggeration={voice.exaggeration} cfg_weight={voice.cfg_weight} "
|
||||
f"speed_factor={voice.speed_factor} seed={voice.seed}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
# Resolve the reference FIRST, before the cache check, so every downstream
|
||||
# decision keys off the exact bytes the cues are cloned from. Two-stage
|
||||
# generation:
|
||||
# 1. VoiceDesign mints a single reference clip in the target persona
|
||||
# (or the user supplies one via --reference-audio).
|
||||
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions
|
||||
# every cue on the reference's speaker embedding.
|
||||
# Without (2), batched generation drifts timbre across cues: a persona
|
||||
# prompt anchors style but not identity, so each batch item picks its
|
||||
# own voice. The reference WAV is cached so subsequent runs only load
|
||||
# the clone model (saves ~20s + 3.4 GB of disk download); when it is
|
||||
# cached, _resolve_reference returns without loading any model, so doing
|
||||
# this ahead of the skip check is cheap.
|
||||
ref_wav_path, ref_text = _resolve_reference(
|
||||
args, audio_dir, instruct, language, reference_text, seed, temperature, top_p
|
||||
)
|
||||
# Pin the cue cache to this exact reference waveform. A re-mint (or a
|
||||
# reference copied in from another storyboard by render.sh) changes these
|
||||
# bytes; without this gate the cue cache would keep cloned-from-the-old-
|
||||
# reference WAVs alongside freshly regenerated ones: two voices in one
|
||||
# video. See _file_sha256.
|
||||
reference_hash = _file_sha256(ref_wav_path)
|
||||
|
||||
index_path = audio_dir / "index.json"
|
||||
|
||||
# Skip generation when the existing audio matches the script: same cue
|
||||
# texts and same gapBeforeMs values in the same order, AND same synth
|
||||
# settings (instruct/language/reference text + reference-WAV hash/model/
|
||||
# seed/temperature/top_p). Saves ~30s of GPU time when iterating on
|
||||
# activity timing without changing narration or persona.
|
||||
if cached_index_matches(
|
||||
index_path,
|
||||
cues,
|
||||
instruct,
|
||||
language,
|
||||
reference_text,
|
||||
args.design_model,
|
||||
args.clone_model,
|
||||
reference_audio_cache_key,
|
||||
reference_hash,
|
||||
seed,
|
||||
temperature,
|
||||
top_p,
|
||||
):
|
||||
reusable = load_reusable_items(index_path, cues, voice)
|
||||
# A fully cached storyboard still has to be re-checked for a rambling cue:
|
||||
# durations live in the index, so spotting one costs no server call, and
|
||||
# falling through to the generation path is what re-rolls its seed.
|
||||
stale = rambling_cues(list(reusable.values()))
|
||||
if len(reusable) == len(cues) and not stale:
|
||||
print(
|
||||
f"[synth] [{args.storyboard}] cached audio matches the current script: skipping generation",
|
||||
f"[synth] [{args.storyboard}] cached audio matches the current script: "
|
||||
"skipping generation",
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
||||
texts = [c["text"].strip() for c in cues]
|
||||
print(
|
||||
f"[synth] cloning {len(texts)} cues from reference (x_vector_only)",
|
||||
flush=True,
|
||||
)
|
||||
for i, t in enumerate(texts):
|
||||
print(f"[synth] {i:2d}: {t}", flush=True)
|
||||
# Only touch the server once we know there is something to synthesize:
|
||||
# a fully cached storyboard must not trigger a 20s checkpoint swap.
|
||||
check_voice_exists(server, voice.voice)
|
||||
ensure_model(server, voice.repo_id)
|
||||
|
||||
clone_model = load_model(args.clone_model, args.device)
|
||||
out_index_base = {
|
||||
"storyboard": args.storyboard,
|
||||
"instruct": instruct,
|
||||
"language": language,
|
||||
"designModel": args.design_model,
|
||||
"cloneModel": args.clone_model,
|
||||
"referenceAudio": reference_audio_cache_key,
|
||||
"referenceText": ref_text,
|
||||
"referenceHash": reference_hash,
|
||||
"seed": seed,
|
||||
"temperature": temperature,
|
||||
"topP": top_p,
|
||||
}
|
||||
reusable = load_reusable_items(
|
||||
index_path,
|
||||
cues,
|
||||
instruct,
|
||||
language,
|
||||
reference_text,
|
||||
args.design_model,
|
||||
args.clone_model,
|
||||
reference_audio_cache_key,
|
||||
reference_hash,
|
||||
seed,
|
||||
temperature,
|
||||
top_p,
|
||||
)
|
||||
out_index_base = {"storyboard": args.storyboard, **voice.fingerprint()}
|
||||
|
||||
def write_index(items: list[dict]) -> None:
|
||||
index_path.write_text(json.dumps({**out_index_base, "items": items}, indent=2))
|
||||
|
||||
items = []
|
||||
for cue_index, cue in enumerate(cues):
|
||||
cached_item = reusable.get(int(cue["cueIndex"]))
|
||||
items: list[dict] = []
|
||||
for cue in cues:
|
||||
cue_index = int(cue["cueIndex"])
|
||||
cached_item = reusable.get(cue_index)
|
||||
if cached_item:
|
||||
items.append(cached_item)
|
||||
write_index(items)
|
||||
print(
|
||||
f"[synth] reusing {cached_item['wav']} {int(cached_item['durationMs']):>5d}ms «{cue['text']}»",
|
||||
f"[synth] reusing {cached_item['wav']} "
|
||||
f"{int(cached_item['durationMs']):>5d}ms «{cue['text']}»",
|
||||
flush=True,
|
||||
)
|
||||
continue
|
||||
|
||||
seed_everything(seed + cue_index)
|
||||
wavs, sr = clone_model.generate_voice_clone(
|
||||
text=[texts[cue_index]],
|
||||
language=language,
|
||||
ref_audio=str(ref_wav_path),
|
||||
ref_text=ref_text,
|
||||
x_vector_only_mode=True,
|
||||
non_streaming_mode=True,
|
||||
do_sample=True,
|
||||
temperature=temperature,
|
||||
top_p=top_p,
|
||||
)
|
||||
if len(wavs) != 1:
|
||||
text = cue["text"].strip()
|
||||
if len(text) > SPLIT_TEXT_WARN_CHARS:
|
||||
print(
|
||||
f"[synth] model returned {len(wavs)} wavs for cue {cue_index}",
|
||||
file=sys.stderr,
|
||||
f"[synth] cue {cue_index} is {len(text)} chars; Chatterbox quality drops on "
|
||||
"long single-pass text. Split the cue in the storyboard.",
|
||||
flush=True,
|
||||
)
|
||||
return 1
|
||||
audio = wavs[0]
|
||||
if hasattr(audio, "cpu"):
|
||||
audio = audio.cpu().float().numpy()
|
||||
wav_name = f"cue_{cue['cueIndex']:03d}.wav"
|
||||
wav_path = audio_dir / wav_name
|
||||
sf.write(str(wav_path), audio, sr)
|
||||
duration_ms = int(round(len(audio) * 1000 / sr))
|
||||
|
||||
# Vary the seed per cue: one seed for every cue in a storyboard makes
|
||||
# consecutive lines land on near-identical intonation contours, which
|
||||
# reads as robotic across a whole video. The voice identity is pinned
|
||||
# by the predefined voice, not the seed, so this costs no consistency.
|
||||
seed = voice.seed + cue_index
|
||||
wav_bytes = generate_cue(server, text, voice, seed)
|
||||
sample_rate, duration_ms = wav_stats(wav_bytes)
|
||||
wav_name = f"cue_{cue_index:03d}.wav"
|
||||
(audio_dir / wav_name).write_bytes(wav_bytes)
|
||||
items.append(
|
||||
{
|
||||
"cueIndex": cue["cueIndex"],
|
||||
"cueIndex": cue_index,
|
||||
"text": cue["text"],
|
||||
"gapBeforeMs": int(cue.get("gapBeforeMs", 0)),
|
||||
"wav": wav_name,
|
||||
"sampleRate": sr,
|
||||
"sampleRate": sample_rate,
|
||||
"durationMs": duration_ms,
|
||||
"seed": seed,
|
||||
}
|
||||
)
|
||||
write_index(items)
|
||||
print(
|
||||
f"[synth] wrote {wav_name} {duration_ms:>5d}ms «{cue['text']}»",
|
||||
flush=True,
|
||||
)
|
||||
print(f"[synth] wrote {wav_name} {duration_ms:>5d}ms «{cue['text']}»", flush=True)
|
||||
|
||||
repair_rambling_cues(server, audio_dir, voice, items)
|
||||
|
||||
write_index(items)
|
||||
total_ms = sum(it["gapBeforeMs"] + it["durationMs"] for it in items)
|
||||
print(
|
||||
f"[synth] [{args.storyboard}] {len(items)} cues, {total_ms}ms of audio (incl. gaps) -> {audio_dir}",
|
||||
f"[synth] [{args.storyboard}] {len(items)} cues, {total_ms}ms of audio "
|
||||
f"(incl. gaps) -> {audio_dir}",
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
|
|
|||
1295
video/tts/uv.lock
generated
1295
video/tts/uv.lock
generated
File diff suppressed because it is too large
Load diff
Loading…
Add table
Add a link
Reference in a new issue