This commit is contained in:
Andras Schmelczer 2026-06-28 11:59:44 +01:00
parent f1601257c7
commit c2070693fb
68 changed files with 2305 additions and 212 deletions

View file

@ -2,7 +2,7 @@
Reads ``output/<storyboard>/narration-script.json`` (emitted by
``dist/preflight.js``) and runs ``Qwen3TTSModel.generate_voice_design`` with
all cue texts as a single batched list that way every cue shares the same
all cue texts as a single batched list: that way every cue shares the same
model state, which keeps prosody and timbre consistent across cues. Per-cue
WAVs and an index manifest go to ``output/<storyboard>/audio/`` for the
recording step (which reads measured cue durations) and the mux step (which
@ -16,7 +16,7 @@ production runs.
We use the VoiceDesign sibling of CustomVoice because it accepts a free-form
voice persona (British accent, narrator register, "no laughter") via the
``instruct`` parameter. CustomVoice's preset speakers are all American or
non-English, and its ``instruct`` is documented for emotion only it
non-English, and its ``instruct`` is documented for emotion only. It
ignored accent directives and bled non-speech tokens (laughter, sighs)
between cues.
@ -43,7 +43,7 @@ from qwen_tts import Qwen3TTSModel
# Two checkpoints: the design model mints the reference clip in the desired
# persona; the clone model conditions every cue on that reference's x-vector.
# Neither CustomVoice nor VoiceDesign support generate_voice_clone — only the
# Neither CustomVoice nor VoiceDesign support generate_voice_clone. Only the
# Base checkpoint does.
DEFAULT_DESIGN_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
DEFAULT_CLONE_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
@ -69,8 +69,8 @@ def _file_sha256(path: Path) -> str:
"""Content hash of a file, used to pin cached cues to a reference WAV.
The cue cache keys off the *settings* that produced the reference
(instruct/seed/), but a re-mint of VoiceDesign or render.sh copying a
different storyboard's reference into this audio dir can swap the actual
(instruct/seed/), but a re-mint of VoiceDesign (or render.sh copying a
different storyboard's reference into this audio dir) can swap the actual
reference waveform out from under those settings. Cloning some cues from
reference A and others from reference B yields two audibly different
speakers in one video. Hashing the bytes of the reference that was
@ -328,7 +328,7 @@ def _resolve_reference(
sf.write(str(ref_wav_path), ref_audio, ref_sr)
ref_meta_path.write_text(json.dumps(ref_meta, indent=2))
# Free the design model before loading the clone model both are 1.7B,
# Free the design model before loading the clone model: both are 1.7B,
# we don't want them resident at the same time.
del design_model
if torch.cuda.is_available():
@ -357,7 +357,7 @@ def main() -> int:
voice = script.get("voice")
if not voice:
print(
f"[synth] {script_path} has no `voice` block — re-run preflight.",
f"[synth] {script_path} has no `voice` block. Re-run preflight.",
file=sys.stderr,
)
return 1
@ -386,7 +386,7 @@ def main() -> int:
# (or the user supplies one via --reference-audio).
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions
# every cue on the reference's speaker embedding.
# Without (2), batched generation drifts timbre across cues a persona
# Without (2), batched generation drifts timbre across cues: a persona
# prompt anchors style but not identity, so each batch item picks its
# own voice. The reference WAV is cached so subsequent runs only load
# the clone model (saves ~20s + 3.4 GB of disk download); when it is
@ -398,13 +398,13 @@ def main() -> int:
# Pin the cue cache to this exact reference waveform. A re-mint (or a
# reference copied in from another storyboard by render.sh) changes these
# bytes; without this gate the cue cache would keep cloned-from-the-old-
# reference WAVs alongside freshly regenerated ones two voices in one
# reference WAVs alongside freshly regenerated ones: two voices in one
# video. See _file_sha256.
reference_hash = _file_sha256(ref_wav_path)
index_path = audio_dir / "index.json"
# Skip generation when the existing audio matches the script same cue
# Skip generation when the existing audio matches the script: same cue
# texts and same gapBeforeMs values in the same order, AND same synth
# settings (instruct/language/reference text + reference-WAV hash/model/
# seed/temperature/top_p). Saves ~30s of GPU time when iterating on
@ -424,7 +424,7 @@ def main() -> int:
top_p,
):
print(
f"[synth] [{args.storyboard}] cached audio matches the current script skipping generation",
f"[synth] [{args.storyboard}] cached audio matches the current script: skipping generation",
flush=True,
)
return 0