wip
This commit is contained in:
parent
f1601257c7
commit
c2070693fb
68 changed files with 2305 additions and 212 deletions
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
Reads ``output/<storyboard>/narration-script.json`` (emitted by
|
||||
``dist/preflight.js``) and runs ``Qwen3TTSModel.generate_voice_design`` with
|
||||
all cue texts as a single batched list — that way every cue shares the same
|
||||
all cue texts as a single batched list: that way every cue shares the same
|
||||
model state, which keeps prosody and timbre consistent across cues. Per-cue
|
||||
WAVs and an index manifest go to ``output/<storyboard>/audio/`` for the
|
||||
recording step (which reads measured cue durations) and the mux step (which
|
||||
|
|
@ -16,7 +16,7 @@ production runs.
|
|||
We use the VoiceDesign sibling of CustomVoice because it accepts a free-form
|
||||
voice persona (British accent, narrator register, "no laughter") via the
|
||||
``instruct`` parameter. CustomVoice's preset speakers are all American or
|
||||
non-English, and its ``instruct`` is documented for emotion only — it
|
||||
non-English, and its ``instruct`` is documented for emotion only. It
|
||||
ignored accent directives and bled non-speech tokens (laughter, sighs)
|
||||
between cues.
|
||||
|
||||
|
|
@ -43,7 +43,7 @@ from qwen_tts import Qwen3TTSModel
|
|||
|
||||
# Two checkpoints: the design model mints the reference clip in the desired
|
||||
# persona; the clone model conditions every cue on that reference's x-vector.
|
||||
# Neither CustomVoice nor VoiceDesign support generate_voice_clone — only the
|
||||
# Neither CustomVoice nor VoiceDesign support generate_voice_clone. Only the
|
||||
# Base checkpoint does.
|
||||
DEFAULT_DESIGN_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
|
||||
DEFAULT_CLONE_MODEL = "Qwen/Qwen3-TTS-12Hz-1.7B-Base"
|
||||
|
|
@ -69,8 +69,8 @@ def _file_sha256(path: Path) -> str:
|
|||
"""Content hash of a file, used to pin cached cues to a reference WAV.
|
||||
|
||||
The cue cache keys off the *settings* that produced the reference
|
||||
(instruct/seed/…), but a re-mint of VoiceDesign — or render.sh copying a
|
||||
different storyboard's reference into this audio dir — can swap the actual
|
||||
(instruct/seed/…), but a re-mint of VoiceDesign (or render.sh copying a
|
||||
different storyboard's reference into this audio dir) can swap the actual
|
||||
reference waveform out from under those settings. Cloning some cues from
|
||||
reference A and others from reference B yields two audibly different
|
||||
speakers in one video. Hashing the bytes of the reference that was
|
||||
|
|
@ -328,7 +328,7 @@ def _resolve_reference(
|
|||
sf.write(str(ref_wav_path), ref_audio, ref_sr)
|
||||
ref_meta_path.write_text(json.dumps(ref_meta, indent=2))
|
||||
|
||||
# Free the design model before loading the clone model — both are 1.7B,
|
||||
# Free the design model before loading the clone model: both are 1.7B,
|
||||
# we don't want them resident at the same time.
|
||||
del design_model
|
||||
if torch.cuda.is_available():
|
||||
|
|
@ -357,7 +357,7 @@ def main() -> int:
|
|||
voice = script.get("voice")
|
||||
if not voice:
|
||||
print(
|
||||
f"[synth] {script_path} has no `voice` block — re-run preflight.",
|
||||
f"[synth] {script_path} has no `voice` block. Re-run preflight.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
|
@ -386,7 +386,7 @@ def main() -> int:
|
|||
# (or the user supplies one via --reference-audio).
|
||||
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions
|
||||
# every cue on the reference's speaker embedding.
|
||||
# Without (2), batched generation drifts timbre across cues — a persona
|
||||
# Without (2), batched generation drifts timbre across cues: a persona
|
||||
# prompt anchors style but not identity, so each batch item picks its
|
||||
# own voice. The reference WAV is cached so subsequent runs only load
|
||||
# the clone model (saves ~20s + 3.4 GB of disk download); when it is
|
||||
|
|
@ -398,13 +398,13 @@ def main() -> int:
|
|||
# Pin the cue cache to this exact reference waveform. A re-mint (or a
|
||||
# reference copied in from another storyboard by render.sh) changes these
|
||||
# bytes; without this gate the cue cache would keep cloned-from-the-old-
|
||||
# reference WAVs alongside freshly regenerated ones — two voices in one
|
||||
# reference WAVs alongside freshly regenerated ones: two voices in one
|
||||
# video. See _file_sha256.
|
||||
reference_hash = _file_sha256(ref_wav_path)
|
||||
|
||||
index_path = audio_dir / "index.json"
|
||||
|
||||
# Skip generation when the existing audio matches the script — same cue
|
||||
# Skip generation when the existing audio matches the script: same cue
|
||||
# texts and same gapBeforeMs values in the same order, AND same synth
|
||||
# settings (instruct/language/reference text + reference-WAV hash/model/
|
||||
# seed/temperature/top_p). Saves ~30s of GPU time when iterating on
|
||||
|
|
@ -424,7 +424,7 @@ def main() -> int:
|
|||
top_p,
|
||||
):
|
||||
print(
|
||||
f"[synth] [{args.storyboard}] cached audio matches the current script — skipping generation",
|
||||
f"[synth] [{args.storyboard}] cached audio matches the current script: skipping generation",
|
||||
flush=True,
|
||||
)
|
||||
return 0
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue