migrate to chatterbox
This commit is contained in:
parent
716e42a57d
commit
988c01c4f7
7 changed files with 480 additions and 1786 deletions
|
|
@ -17,9 +17,10 @@ import { storyboards } from './storyboard.js';
|
|||
* The cue index in each manifest is the source of truth: the runner later
|
||||
* matches storyboard cues to measured durations by index.
|
||||
*/
|
||||
// Em/en-dashes and ellipses make Qwen3-TTS produce dramatic pauses, sighs,
|
||||
// or audible breaths. The captions still render the original (unicode-rich)
|
||||
// text from the storyboard; only the synth input is sanitised.
|
||||
// Em/en-dashes and ellipses read as dramatic pauses, sighs or audible breaths
|
||||
// (originally observed on Qwen3-TTS; kept as a cheap safeguard on Chatterbox).
|
||||
// The captions still render the original (unicode-rich) text from the
|
||||
// storyboard; only the synth input is sanitised.
|
||||
function normalizeForTts(text: string): string {
|
||||
return text
|
||||
.replace(/\s*[\u2014\u2013]\s*/g, ', ')
|
||||
|
|
@ -39,16 +40,17 @@ function emitScript(storyboard: Storyboard): string {
|
|||
gapBeforeMs: cue.gapBeforeMs,
|
||||
}));
|
||||
|
||||
// The voice block is consumed by tts/synth.py. See _resolve_reference and
|
||||
// the cache check there for which fields invalidate cached audio.
|
||||
// The voice block is consumed by tts/synth.py. See VoiceSettings.fingerprint
|
||||
// there for which fields invalidate cached audio.
|
||||
const manifest = {
|
||||
storyboard: storyboard.name,
|
||||
voice: {
|
||||
instruct: storyboard.voice.instruct,
|
||||
voice: storyboard.voice.voice,
|
||||
language: storyboard.voice.language,
|
||||
referenceText: storyboard.voice.referenceText,
|
||||
temperature: storyboard.voice.temperature ?? 0.6,
|
||||
topP: storyboard.voice.topP ?? 0.9,
|
||||
temperature: storyboard.voice.temperature ?? 0.8,
|
||||
exaggeration: storyboard.voice.exaggeration ?? 0.5,
|
||||
cfgWeight: storyboard.voice.cfgWeight ?? 0.5,
|
||||
speedFactor: storyboard.voice.speedFactor ?? 1,
|
||||
seed: storyboard.voice.seed ?? 42,
|
||||
},
|
||||
items,
|
||||
|
|
|
|||
|
|
@ -6,9 +6,8 @@ import type { DashboardRecorder } from './dashboard.js';
|
|||
*
|
||||
* The storyboard is a `Storyboard`: an ordered list of narration cues, each
|
||||
* carrying the activities that play alongside it. Audio is generated FIRST
|
||||
* (one batched Qwen call so the voice stays consistent across cues); the
|
||||
* runner then reads the measured per-cue durations and slots `during`
|
||||
* activities inside each cue's audio window.
|
||||
* (one Chatterbox call per cue); the runner then reads the measured per-cue
|
||||
* durations and slots `during` activities inside each cue's audio window.
|
||||
*
|
||||
* Why cue-anchored: the audio drives pacing. Re-running synth produces a new
|
||||
* set of measured durations and the storyboard self-aligns: you don't have
|
||||
|
|
@ -288,21 +287,29 @@ export interface VideoConfig {
|
|||
posterTimeS: number;
|
||||
}
|
||||
|
||||
/** Qwen3-TTS voice + language settings, sent to synth.py via the narration
|
||||
/** Chatterbox voice + language settings, sent to synth.py via the narration
|
||||
* script. Per storyboard so we can ship a British male narrator on one cut
|
||||
* and a different persona on another. */
|
||||
* and a different persona on another.
|
||||
*
|
||||
* Chatterbox has no persona prompt: the voice IS the accent and the register.
|
||||
* Anything you would once have written as an `instruct` sentence has to be
|
||||
* chosen here as a voice file instead. */
|
||||
export interface VoiceConfig {
|
||||
/** VoiceDesign persona prompt (accent, register, anti-filler directives). */
|
||||
instruct: string;
|
||||
/** Qwen3-TTS language string, e.g. "English". */
|
||||
/** Predefined voice filename on the Chatterbox server, INCLUDING `.wav`
|
||||
* (e.g. "Southern-M.wav"). `GET /v1/audio/voices` lists the valid names. */
|
||||
voice: string;
|
||||
/** ISO code, e.g. "en", "de", "zh". Anything other than "en" makes synth.py
|
||||
* swap the server onto the multilingual checkpoint. */
|
||||
language: string;
|
||||
/** Reference utterance used when minting a generated voice for this language. */
|
||||
referenceText?: string;
|
||||
/** Sampling temperature (default 0.6). */
|
||||
/** Sampling temperature (default 0.8). */
|
||||
temperature?: number;
|
||||
/** Top-p nucleus sampling (default 0.9). */
|
||||
topP?: number;
|
||||
/** Reproducibility seed (default 42). */
|
||||
/** Expressiveness; 0.5 is natural, higher is theatrical (default 0.5). */
|
||||
exaggeration?: number;
|
||||
/** Classifier-free guidance weight (default 0.5). */
|
||||
cfgWeight?: number;
|
||||
/** Post-hoc playback rate (default 1). */
|
||||
speedFactor?: number;
|
||||
/** Reproducibility seed (default 42); synth.py offsets it per cue. */
|
||||
seed?: number;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -38,10 +38,10 @@ type FormFactor = 'desktop' | 'mobile';
|
|||
* to the homepage. The default storyboard is named `recording` so the
|
||||
* existing homepage `/video/recording.mp4` keeps working unchanged.
|
||||
*
|
||||
* Audio is generated first (one batched Qwen call per storyboard, using
|
||||
* its own voice config), so each cue's actual duration is known before
|
||||
* recording. The runner sizes each cue's wall-time to the measured audio
|
||||
* length, padding short `during` blocks with a trailing wait.
|
||||
* Audio is generated first (Chatterbox, per storyboard, using its own voice
|
||||
* config), so each cue's actual duration is known before recording. The
|
||||
* runner sizes each cue's wall-time to the measured audio length, padding
|
||||
* short `during` blocks with a trailing wait.
|
||||
*/
|
||||
|
||||
// School features as served by live /api/features. The data pipeline moved
|
||||
|
|
@ -97,9 +97,12 @@ type RecordingLocale = 'en' | 'de' | 'zh' | 'hi';
|
|||
interface RecordingLocalization {
|
||||
name: string;
|
||||
appLanguage: string;
|
||||
/** ISO code for Chatterbox. Anything but 'en' routes to the multilingual
|
||||
* checkpoint; see tts/synth.py. */
|
||||
ttsLanguage: string;
|
||||
voiceInstruct: string;
|
||||
voiceReferenceText: string;
|
||||
/** Chatterbox predefined voice filename. This is the ONLY accent control we
|
||||
* have, so it stands in for what used to be a written persona prompt. */
|
||||
voice: string;
|
||||
promptText: string;
|
||||
travelTimeLabel: string;
|
||||
exportButtonTitle: string;
|
||||
|
|
@ -134,12 +137,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
en: {
|
||||
name: 'recording',
|
||||
appLanguage: 'en',
|
||||
ttsLanguage: 'English',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful young British male narrator from the North of England with a ' +
|
||||
'strong Manchester accent.',
|
||||
voiceReferenceText:
|
||||
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video.",
|
||||
ttsLanguage: 'en',
|
||||
voice: 'Southern-M.wav',
|
||||
promptText:
|
||||
'First home under £600k, 35 min to central London, good schools, low crime, quiet street, fast broadband',
|
||||
travelTimeLabel: 'Central London',
|
||||
|
|
@ -169,12 +168,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
de: {
|
||||
name: 'recording-de',
|
||||
appLanguage: 'de',
|
||||
ttsLanguage: 'German',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful German male narrator with clear standard German pronunciation ' +
|
||||
'and a friendly, practical delivery.',
|
||||
voiceReferenceText:
|
||||
'Willkommen zur Demonstration. Diese Sprecherstimme hörst du im gesamten Video.',
|
||||
ttsLanguage: 'de',
|
||||
voice: 'German-M.wav',
|
||||
promptText:
|
||||
'Wohnung unter £600k, 35 Min. ins Zentrum von London, gute Schulen, niedrige Kriminalität, ruhige Straßen',
|
||||
travelTimeLabel: 'Zentrum von London',
|
||||
|
|
@ -205,11 +200,8 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
zh: {
|
||||
name: 'recording-zh',
|
||||
appLanguage: 'zh',
|
||||
ttsLanguage: 'Chinese',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful Mandarin Chinese male narrator with clear standard Mandarin ' +
|
||||
'pronunciation and a friendly, practical delivery.',
|
||||
voiceReferenceText: '欢迎观看演示。整段视频都会使用这位旁白的声音。',
|
||||
ttsLanguage: 'zh',
|
||||
voice: 'Mandarin-M.wav',
|
||||
promptText: '60万英镑以内的公寓，35分钟到伦敦市中心，学校好，犯罪率低，街道安静',
|
||||
travelTimeLabel: '伦敦市中心',
|
||||
exportButtonTitle: '导出为 Excel',
|
||||
|
|
@ -235,12 +227,10 @@ const RECORDING_LOCALIZATIONS: Record<RecordingLocale, RecordingLocalization> =
|
|||
hi: {
|
||||
name: 'recording-hi',
|
||||
appLanguage: 'hi',
|
||||
ttsLanguage: 'English',
|
||||
voiceInstruct:
|
||||
'Calm and cheerful Indian male narrator speaking English with a strong Indian accent ' +
|
||||
'and a friendly, practical delivery.',
|
||||
voiceReferenceText:
|
||||
"Welcome to the demonstration. This is the narrator voice you'll hear throughout the video.",
|
||||
// Hindi UI, English narration: the cues below are English on purpose.
|
||||
// Indian-M.wav carries the Indian accent the old persona prompt asked for.
|
||||
ttsLanguage: 'en',
|
||||
voice: 'Indian-M.wav',
|
||||
promptText: 'Flat under £600k, 35 min to central London, good schools, low crime, quiet streets',
|
||||
travelTimeLabel: 'Central London',
|
||||
exportButtonTitle: 'Excel में export करें',
|
||||
|
|
@ -568,11 +558,12 @@ function createRecordingStoryboard(
|
|||
locale: formFactor === 'mobile' ? `${locale}-mobile` : locale,
|
||||
video: buildVideoConfig(formFactor),
|
||||
voice: {
|
||||
instruct: copy.voiceInstruct,
|
||||
voice: copy.voice,
|
||||
language: copy.ttsLanguage,
|
||||
referenceText: copy.voiceReferenceText,
|
||||
temperature: 0.6,
|
||||
topP: 0.9,
|
||||
temperature: 0.8,
|
||||
exaggeration: 0.5,
|
||||
cfgWeight: 0.5,
|
||||
speedFactor: 1,
|
||||
seed: 42,
|
||||
},
|
||||
content: {
|
||||
|
|
@ -715,20 +706,18 @@ const AD_BRAND = {
|
|||
};
|
||||
|
||||
/**
|
||||
* Ad voice persona. The SAME config is used across every ad so the voice
|
||||
* timbre stays consistent across the set (render.sh additionally reuses the
|
||||
* first minted reference WAV for all of them).
|
||||
* Ad voice. The SAME config is used across every ad so the timbre stays
|
||||
* consistent across the set; with Chatterbox that consistency is free, since
|
||||
* a predefined voice is a fixed server-side asset rather than a clip we mint
|
||||
* per render.
|
||||
*/
|
||||
const AD_VOICE = {
|
||||
instruct:
|
||||
'British male creator-style narrator. Warm, confident, conversational, with a ' +
|
||||
'slightly quick pace, like telling a friend about a great find. No salesy hype, ' +
|
||||
'no exaggeration. Short sentences, natural delivery.',
|
||||
language: 'English',
|
||||
referenceText:
|
||||
'This is a short social video for people choosing where to live in England.',
|
||||
temperature: 0.58,
|
||||
topP: 0.9,
|
||||
voice: 'Southern-M.wav',
|
||||
language: 'en',
|
||||
temperature: 0.8,
|
||||
exaggeration: 0.5,
|
||||
cfgWeight: 0.5,
|
||||
speedFactor: 1,
|
||||
seed: 87,
|
||||
};
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue