andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Fix voice

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-27 10:44:57 +01:00
parent e9e47fd811
commit f047e50989
9 changed files with 109 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

12
video/src/storyboard.ts
View file

@ -28,7 +28,7 @@ type FormFactor = 'desktop' | 'mobile';
* real data). On 9:16 cuts the bottom sheet gets dragged out of the way
* whenever the map is the story.
* 4. Filter names MUST match live /api/features exactly (e.g.
* "Serious crime (avg/yr)", "Distance to nearest amenity (Waitrose)
* "Serious crime (/yr, 7y)", "Distance to nearest amenity (Waitrose)
* (km)") — wrong names silently no-op and the map never changes.
* preflight.ts validates every stubbed name against the live API.
*
@ -592,7 +592,7 @@ function createRecordingStoryboard(
// Loose enough to keep the central-London map richly populated — a
// cap of 20 emptied the city centre and left the zoom with nothing
// to land on.
'Serious crime (avg/yr)': [0, 40],
'Serious crime (/yr, 7y)': [0, 40],
[SCHOOL_GOOD_PRIMARY]: [1, 10],
'Noise (dB)': [0, 65],
// ≥30 Mbps, not ≥100: FTTP coverage gaps make a 100 floor empty
@ -888,7 +888,7 @@ const AD_CONFIGS: DemoAdStoryboardConfig[] = [
filters: {
'Property type': ['Flats/Maisonettes'],
'Estimated current price': [0, 600000],
'Serious crime (avg/yr)': [0, 35],
'Serious crime (/yr, 7y)': [0, 35],
'Noise (dB)': [0, 60],
},
travelTimeFilters: [
@ -1038,7 +1038,7 @@ const AD_CONFIGS: DemoAdStoryboardConfig[] = [
promptText: 'Quiet London street under 55 decibels, low crime',
filters: {
'Noise (dB)': [0, 55],
'Serious crime (avg/yr)': [0, 35],
'Serious crime (/yr, 7y)': [0, 35],
},
initialZoom: 10.6,
posterTimeS: 7,
@ -1076,7 +1076,7 @@ const AD_CONFIGS: DemoAdStoryboardConfig[] = [
filters: {
'Estimated current price': [0, 350000],
[SCHOOL_GOOD_PRIMARY]: [2, 10],
'Serious crime (avg/yr)': [0, 30],
'Serious crime (/yr, 7y)': [0, 30],
},
initialZoom: 11.0,
posterTimeS: 13,
@ -1194,7 +1194,7 @@ const AD_CONFIGS: DemoAdStoryboardConfig[] = [
promptText: 'Under £500k, 35 min to central London, low crime, good schools',
filters: {
'Estimated current price': [0, 500000],
'Serious crime (avg/yr)': [0, 35],
'Serious crime (/yr, 7y)': [0, 35],
[SCHOOL_GOOD_PRIMARY]: [1, 10],
},
travelTimeFilters: [

100
video/tts/synth.py
View file

@ -28,6 +28,7 @@ Run from the ``video/`` directory:
from __future__ import annotations
import argparse
import hashlib
import json
import os
import random
@ -64,6 +65,25 @@ def _safe_load_json(path: Path) -> object | None:
return None
def _file_sha256(path: Path) -> str:
"""Content hash of a file, used to pin cached cues to a reference WAV.
The cue cache keys off the *settings* that produced the reference
(instruct/seed/…), but a re-mint of VoiceDesign — or render.sh copying a
different storyboard's reference into this audio dir — can swap the actual
reference waveform out from under those settings. Cloning some cues from
reference A and others from reference B yields two audibly different
speakers in one video. Hashing the bytes of the reference that was
actually used closes that gap: any change to the reference invalidates
every cue, so all cues in a render share one timbre.
"""
h = hashlib.sha256()
with path.open("rb") as fh:
for chunk in iter(lambda: fh.read(1 << 20), b""):
h.update(chunk)
return h.hexdigest()
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
@ -120,6 +140,7 @@ def cached_index_matches(
design_model: str,
clone_model: str,
reference_audio: str,
reference_hash: str,
seed: int,
temperature: float,
top_p: float,
@ -127,10 +148,11 @@ def cached_index_matches(
"""Return True iff index_path's cue list lines up with `cues` 1:1.
Compared fields: ``cueIndex``, ``text``, ``gapBeforeMs`` plus the synth
settings (``instruct``, ``language``, reference text, models, ``seed``,
``temperature``, ``top_p``).
settings (``instruct``, ``language``, reference text, models, the hash of
the reference WAV the cues were cloned from, ``seed``, ``temperature``,
``top_p``).
All cue WAV files must also exist on disk. Mismatched length, reordered
cues, or a missing WAV invalidate the cache.
cues, a swapped reference waveform, or a missing WAV invalidate the cache.
"""
if not index_path.exists():
return False
@ -146,6 +168,8 @@ def cached_index_matches(
return False
if cached.get("referenceAudio", "") != reference_audio:
return False
if cached.get("referenceHash", "") != reference_hash:
return False
if int(cached.get("seed", -1)) != seed:
return False
if float(cached.get("temperature", -1)) != temperature:
@ -177,6 +201,7 @@ def load_reusable_items(
design_model: str,
clone_model: str,
reference_audio: str,
reference_hash: str,
seed: int,
temperature: float,
top_p: float,
@ -184,7 +209,10 @@ def load_reusable_items(
"""Return cue-indexed cached items that match the current synth settings.
Unlike ``cached_index_matches`` this accepts a partial index, so a long
CPU synthesis run can be resumed cue-by-cue after an interruption.
CPU synthesis run can be resumed cue-by-cue after an interruption. The
reference-hash gate is what stops a re-mint from leaving some cues cloned
from the previous reference (a second voice) while only the edited cues
regenerate from the new one.
"""
if not index_path.exists():
return {}
@ -200,6 +228,8 @@ def load_reusable_items(
return {}
if cached.get("referenceAudio", "") != reference_audio:
return {}
if cached.get("referenceHash", "") != reference_hash:
return {}
if int(cached.get("seed", -1)) != seed:
return {}
if float(cached.get("temperature", -1)) != temperature:
@ -343,13 +373,44 @@ def main() -> int:
audio_dir.mkdir(parents=True, exist_ok=True)
print(f"[synth] [{args.storyboard}] persona: {instruct}", flush=True)
print(
f"[synth] [{args.storyboard}] sampling: temperature={temperature} top_p={top_p} seed={seed} language={language}",
flush=True,
)
# Resolve the reference FIRST, before the cache check, so every downstream
# decision keys off the exact bytes the cues are cloned from. Two-stage
# generation:
# 1. VoiceDesign mints a single reference clip in the target persona
# (or the user supplies one via --reference-audio).
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions
# every cue on the reference's speaker embedding.
# Without (2), batched generation drifts timbre across cues — a persona
# prompt anchors style but not identity, so each batch item picks its
# own voice. The reference WAV is cached so subsequent runs only load
# the clone model (saves ~20s + 3.4 GB of disk download); when it is
# cached, _resolve_reference returns without loading any model, so doing
# this ahead of the skip check is cheap.
ref_wav_path, ref_text = _resolve_reference(
args, audio_dir, instruct, language, reference_text, seed, temperature, top_p
)
# Pin the cue cache to this exact reference waveform. A re-mint (or a
# reference copied in from another storyboard by render.sh) changes these
# bytes; without this gate the cue cache would keep cloned-from-the-old-
# reference WAVs alongside freshly regenerated ones — two voices in one
# video. See _file_sha256.
reference_hash = _file_sha256(ref_wav_path)
index_path = audio_dir / "index.json"
# Skip generation when the existing audio matches the script — same cue
# texts and same gapBeforeMs values in the same order, AND same synth
# settings (instruct/language/reference/model/seed/temperature/top_p).
# Saves ~30s of GPU time when iterating on activity timing without
# changing narration or persona.
# settings (instruct/language/reference text + reference-WAV hash/model/
# seed/temperature/top_p). Saves ~30s of GPU time when iterating on
# activity timing without changing narration or persona.
if cached_index_matches(
audio_dir / "index.json",
index_path,
cues,
instruct,
language,
@ -357,6 +418,7 @@ def main() -> int:
args.design_model,
args.clone_model,
reference_audio_cache_key,
reference_hash,
seed,
temperature,
top_p,
@ -368,25 +430,6 @@ def main() -> int:
return 0
texts = [c["text"].strip() for c in cues]
print(f"[synth] [{args.storyboard}] persona: {instruct}", flush=True)
print(
f"[synth] [{args.storyboard}] sampling: temperature={temperature} top_p={top_p} seed={seed} language={language}",
flush=True,
)
# Two-stage generation:
# 1. VoiceDesign mints a single reference clip in the target persona
# (or the user supplies one via --reference-audio).
# 2. Base + generate_voice_clone(x_vector_only_mode=True) conditions
# every cue on the reference's speaker embedding.
# Without (2), batched generation drifts timbre across cues — a persona
# prompt anchors style but not identity, so each batch item picks its
# own voice. The reference WAV is cached so subsequent runs only load
# the clone model (saves ~20s + 3.4 GB of disk download).
ref_wav_path, ref_text = _resolve_reference(
args, audio_dir, instruct, language, reference_text, seed, temperature, top_p
)
print(
f"[synth] cloning {len(texts)} cues from reference (x_vector_only)",
flush=True,
@ -403,11 +446,11 @@ def main() -> int:
"cloneModel": args.clone_model,
"referenceAudio": reference_audio_cache_key,
"referenceText": ref_text,
"referenceHash": reference_hash,
"seed": seed,
"temperature": temperature,
"topP": top_p,
}
index_path = audio_dir / "index.json"
reusable = load_reusable_items(
index_path,
cues,
@ -417,6 +460,7 @@ def main() -> int:
args.design_model,
args.clone_model,
reference_audio_cache_key,
reference_hash,
seed,
temperature,
top_p,