diff --git a/src/audio/garden-audio-config.ts b/src/audio/garden-audio-config.ts new file mode 100644 index 0000000..b5fbc1d --- /dev/null +++ b/src/audio/garden-audio-config.ts @@ -0,0 +1,127 @@ +import type { PianoNoteRole } from './garden-audio-types'; + +export const DEFAULT_AUDIO_VOLUME = 0.5; +export const SILENT_AUDIO_GAIN = 0.0001; + +type GardenAudioChordQuality = 'major' | 'minor' | 'sus2' | 'sus4'; + +export interface GardenAudioChord { + rootOffset: number; + quality: GardenAudioChordQuality; +} + +export interface GardenAudioVibeSettings { + idleIntensity: number; + bpm: number; + rampUpIntensity: number; + rampUpTime: number; + noteLength: number; + notePitchOffset: number; + brightness: number; + scale?: Array; + progression?: Array; +} + +export interface GardenAudioVibeProfile extends GardenAudioVibeSettings { + rootMidi: number; + scale: Array; + progression: Array; +} + +export const defaultGardenAudioVibeSettings: GardenAudioVibeSettings = { + idleIntensity: 0.08, + bpm: 74, + rampUpIntensity: 0.85, + rampUpTime: 0.08, + noteLength: 0.42, + notePitchOffset: 0, + brightness: 1, +}; + +export const createGardenAudioConfig = () => ({ + masterVolume: DEFAULT_AUDIO_VOLUME, + fadeInSeconds: 0.45, + updateRampSeconds: 0.08, + delay: { + timeBeats: 0.5, + timeMinSeconds: 0.18, + timeMaxSeconds: 0.72, + feedback: 0.12, + wetGain: 0.044, + erasingActivity: 0.12, + activityFeedbackWeight: 0.08, + feedbackMax: 0.32, + feedbackMin: 0.04, + outputActivityWeight: 0.5, + outputBase: 0.65, + outputActivityDuck: 0.28, + timeRampSeconds: 0.12, + }, + piano: { + maxVoices: 24, + gain: 0.48, + sustainSeconds: 0.42, + sustainLevel: 0.26, + releaseSeconds: 0.34, + lowpassHz: 7000, + gainAttackSeconds: 0.006, + lowpassMaxHz: 12000, + lowpassMinHz: 1400, + sustainBase: 0.45, + sustainVelocityRange: 0.55, + }, + rhythm: { + idleIntensity: defaultGardenAudioVibeSettings.idleIntensity, + bpm: defaultGardenAudioVibeSettings.bpm, + stepsPerBeat: 4, + stepsPerBar: 16, + sparseActivity: 0.055, + }, + eraser: { + minIntervalSeconds: 0.12, + noiseGain: 0.028, + filterMinHz: 650, + filterMaxHz: 3600, + durationSeconds: 0.08, + pan: 0, + pianoActivity: 0, + }, + energy: { + decaySeconds: 0.9, + releaseSeconds: 1.15, + strokeDecaySeconds: 0.32, + }, + graph: { + pianoBusGains: { + pad: 0.86, + support: 0.94, + texture: 0.88, + gesture: 1, + brush: 0.9, + stinger: 0.92, + } satisfies Record, + pianoBusActivityDucking: { + pad: 0.42, + support: 0.18, + texture: -0.06, + gesture: 0, + brush: -0.08, + stinger: 0, + } satisfies Record, + noiseBusGain: 0.72, + }, + input: { + fullActivitySpeed: 0.86, + activityNoiseFloorSpeed: 0.025, + activityCurve: 0.74, + activitySoftCeiling: 0.96, + activityAttackSeconds: 0.055, + activityReleaseSeconds: 0.2, + minAudibleDistance: 0.0025, + manicActivityThreshold: 0.9, + manicReleaseThreshold: 0.76, + maniaSmoothingSeconds: 0.12, + }, +}); + +export type GardenAudioConfig = ReturnType; diff --git a/src/audio/garden-audio-energy.ts b/src/audio/garden-audio-energy.ts new file mode 100644 index 0000000..7df3eb8 --- /dev/null +++ b/src/audio/garden-audio-energy.ts @@ -0,0 +1,66 @@ +import { approach, clamp01 } from '../utils/math'; +import type { GardenAudioConfig, GardenAudioVibeProfile } from './garden-audio-config'; + +export class GardenAudioEnergy { + private isGestureActive = false; + private energy = 0; + private targetEnergy = 0; + private lastEnergyUpdateAt = 0; + + public constructor(private readonly config: GardenAudioConfig) {} + + public beginGesture(now: number): void { + this.isGestureActive = true; + this.lastEnergyUpdateAt = now; + } + + public endGesture(): void { + this.isGestureActive = false; + this.targetEnergy = 0; + } + + public recordStroke(strokeEnergy: number, profile: GardenAudioVibeProfile): void { + this.targetEnergy = Math.max(this.targetEnergy, strokeEnergy); + if (this.isGestureActive) { + this.energy = Math.max(this.energy, strokeEnergy * profile.rampUpIntensity); + } + } + + public silence(): void { + this.targetEnergy = 0; + this.energy = 0; + } + + public update(now: number, profile: GardenAudioVibeProfile): void { + if (this.lastEnergyUpdateAt <= 0) { + this.lastEnergyUpdateAt = now; + return; + } + + const elapsedSeconds = now - this.lastEnergyUpdateAt; + this.lastEnergyUpdateAt = now; + this.targetEnergy *= Math.exp( + -elapsedSeconds / this.config.energy.strokeDecaySeconds + ); + + const target = this.isGestureActive ? this.targetEnergy : 0; + let timeConstant = this.config.energy.decaySeconds; + if (!this.isGestureActive) { + timeConstant = this.config.energy.releaseSeconds; + } else if (target > this.energy) { + timeConstant = profile.rampUpTime; + } + this.energy = approach(this.energy, target, elapsedSeconds, timeConstant); + } + + public getLevel(): number { + return clamp01(this.energy); + } + + public reset(): void { + this.isGestureActive = false; + this.energy = 0; + this.targetEnergy = 0; + this.lastEnergyUpdateAt = 0; + } +} diff --git a/src/audio/garden-audio-gesture-state.ts b/src/audio/garden-audio-gesture-state.ts new file mode 100644 index 0000000..abd27f7 --- /dev/null +++ b/src/audio/garden-audio-gesture-state.ts @@ -0,0 +1,75 @@ +import { approach, clamp, clamp01, smoothstep } from '../utils/math'; +import type { GardenAudioConfig } from './garden-audio-config'; +import type { GardenAudioStrokeMetrics } from './garden-audio-input'; + +interface GardenAudioGestureFrame { + activity: number; + maniaAmount: number; +} + +export class GardenAudioGestureState { + private activity = 0; + private maniaAmount = 0; + private isManic = false; + + public constructor(private readonly inputConfig: GardenAudioConfig['input']) {} + + public recordStroke({ + metrics, + }: { + metrics: GardenAudioStrokeMetrics; + }): GardenAudioGestureFrame { + const targetActivity = this.getTargetActivity(metrics); + const activityTimeConstant = + targetActivity > this.activity + ? this.inputConfig.activityAttackSeconds + : this.inputConfig.activityReleaseSeconds; + this.activity = approach( + this.activity, + targetActivity, + metrics.elapsedSeconds, + activityTimeConstant + ); + + if (this.activity >= this.inputConfig.manicActivityThreshold) { + this.isManic = true; + } else if (this.activity <= this.inputConfig.manicReleaseThreshold) { + this.isManic = false; + } + + const maniaTarget = this.isManic + ? smoothstep(this.inputConfig.manicReleaseThreshold, 1, this.activity) + : 0; + this.maniaAmount = approach( + this.maniaAmount, + maniaTarget, + metrics.elapsedSeconds, + this.inputConfig.maniaSmoothingSeconds + ); + + return { + activity: this.activity, + maniaAmount: this.maniaAmount, + }; + } + + public reset(): void { + this.activity = 0; + this.maniaAmount = 0; + this.isManic = false; + } + + private getTargetActivity(metrics: GardenAudioStrokeMetrics): number { + const speedRange = + this.inputConfig.fullActivitySpeed - this.inputConfig.activityNoiseFloorSpeed; + const speedAmount = clamp01( + (metrics.normalizedSpeed - this.inputConfig.activityNoiseFloorSpeed) / speedRange + ); + const distanceAmount = clamp01( + metrics.normalizedDistance / this.inputConfig.minAudibleDistance + ); + const activity = Math.pow(speedAmount, this.inputConfig.activityCurve); + + return clamp(activity * distanceAmount, 0, this.inputConfig.activitySoftCeiling); + } +} diff --git a/src/audio/garden-audio-graph.ts b/src/audio/garden-audio-graph.ts new file mode 100644 index 0000000..a288465 --- /dev/null +++ b/src/audio/garden-audio-graph.ts @@ -0,0 +1,347 @@ +import { clamp } from '../utils/math'; +import { SILENT_AUDIO_GAIN, type GardenAudioConfig } from './garden-audio-config'; +import type { PianoNoteRole } from './garden-audio-types'; + +type AudioSessionType = NonNullable['type']; + +type NavigatorWithAudioSession = Navigator & { + audioSession?: { + type: + | 'auto' + | 'playback' + | 'ambient' + | 'transient' + | 'transient-solo' + | 'play-and-record'; + }; +}; + +const outputHighPassFrequencyHz = 45; +const noiseBufferDurationSeconds = 1; +const graphTuning = { + closeGain: SILENT_AUDIO_GAIN, + closeRampSeconds: 0.015, + delayMaxSeconds: 2, + eventBusGain: 1, + noiseMax: 1, + noiseMin: -1, + latencyHint: 'interactive', + outputFilterType: 'highpass', + compressor: { + thresholdDb: -18, + kneeDb: 18, + ratio: 2.1, + attackSeconds: 0.018, + releaseSeconds: 0.18, + }, +} as const; +const delayFilterTuning = { + feedbackHighPassHz: 180, + feedbackLowPassHz: 5200, + returnLowPassHz: 6200, +}; + +export class GardenAudioGraph { + public context: AudioContext | null = null; + public eventBus: GainNode | null = null; + public delayInput: GainNode | null = null; + public noiseBus: GainNode | null = null; + public noiseBuffer: AudioBuffer | null = null; + + private masterGain: GainNode | null = null; + private delayNode: DelayNode | null = null; + private delayFeedback: GainNode | null = null; + private delayOutput: GainNode | null = null; + private lastPianoBusActivity = 0; + private pianoBusGainScale = 1; + private pianoBusGainScaleAutomationUntil = 0; + private pianoBusGainScaleTimeConstantSeconds = 0; + private previousAudioSessionType: AudioSessionType | null = null; + private readonly pianoBuses = new Map(); + + public constructor(private readonly config: GardenAudioConfig) {} + + public ensureContext(canCreate: boolean): AudioContext | null { + if (this.context) { + return this.context; + } + + if (!canCreate) { + return null; + } + + const AudioContextConstructor = globalThis.AudioContext; + if (!AudioContextConstructor) { + return null; + } + + // Tells iOS to treat this as media playback, so the hardware ringer/mute + // switch does not silence Web Audio output. No-op on browsers without the + // Audio Session API. + const audioSession = (navigator as NavigatorWithAudioSession).audioSession; + if (audioSession) { + this.previousAudioSessionType ??= audioSession.type; + audioSession.type = 'playback'; + } + + const context = new AudioContextConstructor({ + latencyHint: graphTuning.latencyHint, + }); + const masterGain = context.createGain(); + const highPass = context.createBiquadFilter(); + const compressor = context.createDynamicsCompressor(); + + masterGain.gain.value = 0; + highPass.type = graphTuning.outputFilterType; + highPass.frequency.value = outputHighPassFrequencyHz; + compressor.threshold.value = graphTuning.compressor.thresholdDb; + compressor.knee.value = graphTuning.compressor.kneeDb; + compressor.ratio.value = graphTuning.compressor.ratio; + compressor.attack.value = graphTuning.compressor.attackSeconds; + compressor.release.value = graphTuning.compressor.releaseSeconds; + + masterGain.connect(highPass); + highPass.connect(compressor); + compressor.connect(context.destination); + + this.context = context; + this.masterGain = masterGain; + this.noiseBuffer = this.createNoiseBuffer(context); + this.createDelay(context, masterGain); + this.createBuses(context, masterGain); + + return context; + } + + public setMasterGain(targetGain: number, timeConstantSeconds: number): void { + if (!this.context || !this.masterGain) { + return; + } + + this.masterGain.gain.setTargetAtTime( + targetGain, + this.context.currentTime, + timeConstantSeconds + ); + } + + public applyDelayProfile(bpm: number): void { + if (!this.context || !this.delayNode) { + return; + } + + this.delayNode.delayTime.setTargetAtTime( + this.getDelayTimeSecondsForBpm(bpm), + this.context.currentTime, + this.config.delay.timeRampSeconds + ); + } + + public updateDelay(activity: number, bpm: number): void { + if (!this.context || !this.delayNode || !this.delayFeedback || !this.delayOutput) { + return; + } + + const now = this.context.currentTime; + const normalizedActivity = clamp(activity, 0, 1); + this.delayNode.delayTime.setTargetAtTime( + this.getDelayTimeSecondsForBpm(bpm), + now, + this.config.delay.timeRampSeconds + ); + this.delayFeedback.gain.setTargetAtTime( + clamp( + this.config.delay.feedback + + normalizedActivity * this.config.delay.activityFeedbackWeight, + this.config.delay.feedbackMin, + this.config.delay.feedbackMax + ), + now, + this.config.updateRampSeconds + ); + this.delayOutput.gain.setTargetAtTime( + this.config.delay.wetGain * + (this.config.delay.outputBase + + normalizedActivity * this.config.delay.outputActivityWeight) * + (1 - normalizedActivity * this.config.delay.outputActivityDuck), + now, + this.config.updateRampSeconds + ); + this.updatePianoBusGains(normalizedActivity, now); + } + + public getPianoBus(role: PianoNoteRole | undefined): GainNode | null { + return this.pianoBuses.get(role ?? 'gesture') ?? this.eventBus; + } + + public setPianoBusGainScale(targetScale: number, timeConstantSeconds: number): void { + if (!this.context) { + this.pianoBusGainScale = clamp(targetScale, 0, 1); + return; + } + + const now = this.context.currentTime; + + this.pianoBusGainScale = clamp(targetScale, 0, 1); + this.pianoBusGainScaleTimeConstantSeconds = timeConstantSeconds; + this.pianoBusGainScaleAutomationUntil = now + timeConstantSeconds * 4; + this.updatePianoBusGains(this.lastPianoBusActivity, now, timeConstantSeconds); + } + + public async close(): Promise { + const context = this.context; + if (!context) { + return; + } + + if (this.masterGain && context.state !== 'closed') { + this.masterGain.gain.setTargetAtTime( + graphTuning.closeGain, + context.currentTime, + graphTuning.closeRampSeconds + ); + } + + this.clearNodes(); + + if (context.state !== 'closed') { + await context.close().catch(() => undefined); + } + + this.restoreAudioSessionType(); + } + + private restoreAudioSessionType(): void { + const previousType = this.previousAudioSessionType; + this.previousAudioSessionType = null; + if (previousType === null) { + return; + } + + const audioSession = (navigator as NavigatorWithAudioSession).audioSession; + if (audioSession) { + audioSession.type = previousType; + } + } + + private createDelay(context: AudioContext, masterGain: GainNode): void { + const delayInput = context.createGain(); + const delayNode = context.createDelay(graphTuning.delayMaxSeconds); + const delayFeedback = context.createGain(); + const delayOutput = context.createGain(); + const feedbackHighPass = context.createBiquadFilter(); + const feedbackLowPass = context.createBiquadFilter(); + const returnLowPass = context.createBiquadFilter(); + + delayNode.delayTime.value = this.getDelayTimeSecondsForBpm(this.config.rhythm.bpm); + delayFeedback.gain.value = this.config.delay.feedback; + delayOutput.gain.value = this.config.delay.wetGain; + feedbackHighPass.type = 'highpass'; + feedbackHighPass.frequency.value = delayFilterTuning.feedbackHighPassHz; + feedbackLowPass.type = 'lowpass'; + feedbackLowPass.frequency.value = delayFilterTuning.feedbackLowPassHz; + returnLowPass.type = 'lowpass'; + returnLowPass.frequency.value = delayFilterTuning.returnLowPassHz; + + delayInput.connect(delayNode); + delayNode.connect(feedbackHighPass); + feedbackHighPass.connect(feedbackLowPass); + feedbackLowPass.connect(delayFeedback); + delayFeedback.connect(delayNode); + delayNode.connect(returnLowPass); + returnLowPass.connect(delayOutput); + delayOutput.connect(masterGain); + + this.delayInput = delayInput; + this.delayNode = delayNode; + this.delayFeedback = delayFeedback; + this.delayOutput = delayOutput; + } + + private createBuses(context: AudioContext, masterGain: GainNode): void { + const eventBus = context.createGain(); + eventBus.gain.value = graphTuning.eventBusGain; + eventBus.connect(masterGain); + this.eventBus = eventBus; + this.pianoBuses.clear(); + + (Object.keys(this.config.graph.pianoBusGains) as Array).forEach( + (role) => { + const bus = context.createGain(); + bus.gain.value = this.config.graph.pianoBusGains[role]; + bus.connect(eventBus); + this.pianoBuses.set(role, bus); + } + ); + + this.noiseBus = context.createGain(); + this.noiseBus.gain.value = this.config.graph.noiseBusGain; + this.noiseBus.connect(eventBus); + } + + private updatePianoBusGains( + activity: number, + now: number, + timeConstantSeconds?: number + ): void { + const effectiveTimeConstantSeconds = + timeConstantSeconds ?? + (now < this.pianoBusGainScaleAutomationUntil + ? this.pianoBusGainScaleTimeConstantSeconds + : this.config.updateRampSeconds); + + this.lastPianoBusActivity = activity; + this.pianoBuses.forEach((bus, role) => { + const baseGain = this.config.graph.pianoBusGains[role]; + const ducking = this.config.graph.pianoBusActivityDucking[role]; + bus.gain.setTargetAtTime( + Math.max(0, baseGain * (1 - activity * ducking) * this.pianoBusGainScale), + now, + effectiveTimeConstantSeconds + ); + }); + } + + private getDelayTimeSecondsForBpm(bpm: number): number { + const safeBpm = Number.isFinite(bpm) ? Math.max(1, bpm) : this.config.rhythm.bpm; + return clamp( + (60 / safeBpm) * this.config.delay.timeBeats, + this.config.delay.timeMinSeconds, + this.config.delay.timeMaxSeconds + ); + } + + private createNoiseBuffer(context: AudioContext): AudioBuffer { + const buffer = context.createBuffer( + 1, + Math.floor(context.sampleRate * noiseBufferDurationSeconds), + context.sampleRate + ); + const data = buffer.getChannelData(0); + + for (let index = 0; index < data.length; index++) { + data[index] = + graphTuning.noiseMin + + Math.random() * (graphTuning.noiseMax - graphTuning.noiseMin); + } + + return buffer; + } + + private clearNodes(): void { + this.context = null; + this.eventBus = null; + this.delayInput = null; + this.noiseBus = null; + this.noiseBuffer = null; + this.masterGain = null; + this.delayNode = null; + this.delayFeedback = null; + this.delayOutput = null; + this.lastPianoBusActivity = 0; + this.pianoBusGainScale = 1; + this.pianoBusGainScaleAutomationUntil = 0; + this.pianoBusGainScaleTimeConstantSeconds = 0; + this.pianoBuses.clear(); + } +} diff --git a/src/audio/garden-audio-input.ts b/src/audio/garden-audio-input.ts new file mode 100644 index 0000000..4c67507 --- /dev/null +++ b/src/audio/garden-audio-input.ts @@ -0,0 +1,27 @@ +import type { GardenAudioStroke } from './garden-audio-types'; + +const minElapsedSeconds = 0.001; + +export interface GardenAudioStrokeMetrics { + elapsedSeconds: number; + normalizedDistance: number; + normalizedSpeed: number; +} + +export const getStrokeMetrics = (stroke: GardenAudioStroke): GardenAudioStrokeMetrics => { + const dx = stroke.to[0] - stroke.from[0]; + const dy = stroke.to[1] - stroke.from[1]; + const distancePixels = Math.hypot(dx, dy); + const elapsedSeconds = Math.max(minElapsedSeconds, stroke.elapsedSeconds ?? 0); + const normalizationPixels = Math.max( + 1, + Math.min(stroke.canvasSize[0], stroke.canvasSize[1]) + ); + const normalizedDistance = distancePixels / normalizationPixels; + + return { + elapsedSeconds, + normalizedDistance, + normalizedSpeed: normalizedDistance / elapsedSeconds, + }; +}; diff --git a/src/audio/garden-audio-music.ts b/src/audio/garden-audio-music.ts new file mode 100644 index 0000000..a0e5e7b --- /dev/null +++ b/src/audio/garden-audio-music.ts @@ -0,0 +1,33 @@ +import type { VibePreset } from '../vibes'; +import type { GardenAudioChord, GardenAudioVibeProfile } from './garden-audio-config'; + +export const PITCH_SEMITONES_PER_OCTAVE = 12; + +const DEFAULT_PROGRESSION: ReadonlyArray = [ + { rootOffset: 0, quality: 'major' }, + { rootOffset: 9, quality: 'minor' }, + { rootOffset: 5, quality: 'major' }, + { rootOffset: 7, quality: 'major' }, +]; + +const DEFAULT_ROOT_MIDI = 57; +const DEFAULT_SCALE: ReadonlyArray = [0, 2, 4, 7, 9]; + +const getProfileScale = (vibe: VibePreset): Array => { + const scale = vibe.audio.scale?.length ? vibe.audio.scale : DEFAULT_SCALE; + return [...scale]; +}; + +const getProfileProgression = (vibe: VibePreset): Array => + (vibe.audio.progression?.length ? vibe.audio.progression : DEFAULT_PROGRESSION).map( + (chord) => ({ ...chord }) + ); + +export const getVibeProfile = (vibe: VibePreset): GardenAudioVibeProfile => { + return { + ...vibe.audio, + rootMidi: DEFAULT_ROOT_MIDI + vibe.audio.notePitchOffset, + scale: getProfileScale(vibe), + progression: getProfileProgression(vibe), + }; +}; diff --git a/src/audio/garden-audio-types.ts b/src/audio/garden-audio-types.ts new file mode 100644 index 0000000..fecbcf8 --- /dev/null +++ b/src/audio/garden-audio-types.ts @@ -0,0 +1,48 @@ +import type { VibePreset } from '../vibes'; + +export interface GardenAudioSnapshot { + vibe: VibePreset; + isErasing: boolean; +} + +export interface GardenAudioStroke { + vibe: VibePreset; + from: ArrayLike; + to: ArrayLike; + canvasSize: ArrayLike; + isErasing: boolean; + elapsedSeconds: number; +} + +export interface LoadedPianoSample { + midi: number; + buffer: AudioBuffer; +} + +export interface PianoNote { + midi: number; + velocity: number; + startTime: number; + durationSeconds: number; + pan: number; + role?: PianoNoteRole; + delaySend?: number; + lowpassHz?: number; + sustainSeconds?: number; +} + +export type PianoNoteRole = + | 'pad' + | 'support' + | 'texture' + | 'gesture' + | 'brush' + | 'stinger'; + +export interface NoiseBurst { + startTime: number; + durationSeconds: number; + gain: number; + filterHz: number; + pan: number; +} diff --git a/src/audio/garden-audio.ts b/src/audio/garden-audio.ts new file mode 100644 index 0000000..3aa4c88 --- /dev/null +++ b/src/audio/garden-audio.ts @@ -0,0 +1,448 @@ +import { ErrorHandler, Severity } from '../utils/error-handler'; +import { clamp01 } from '../utils/math'; +import type { VibeId, VibePreset } from '../vibes'; +import { + SILENT_AUDIO_GAIN, + type GardenAudioConfig, + type GardenAudioVibeProfile, +} from './garden-audio-config'; +import { GardenAudioEnergy } from './garden-audio-energy'; +import { GardenAudioGestureState } from './garden-audio-gesture-state'; +import { GardenAudioGraph } from './garden-audio-graph'; +import { getStrokeMetrics } from './garden-audio-input'; +import { getVibeProfile } from './garden-audio-music'; +import type { GardenAudioSnapshot, GardenAudioStroke } from './garden-audio-types'; +import { GenerativePianoEngine } from './generative-piano'; +import { NoiseBurstPlayer } from './noise-burst-player'; +import { PianoSampler } from './piano-sampler'; + +type AudioLifecycle = 'idle' | 'started' | 'destroyed'; +type PianoReleasePhase = + | { kind: 'idle' } + | { kind: 'awaiting-fade' } + | { kind: 'scheduled-fade'; fadeAt: number } + | { kind: 'settling'; stopAt: number }; + +const muteRampSeconds = 0.02; +const brushUpPianoBusFadeSeconds = 2.4; +const brushUpPianoBusFadeSettleSeconds = 3.2; +const vibeChangeStingerMinIntervalSeconds = 0.45; + +export class GardenAudio { + private readonly graph: GardenAudioGraph; + private readonly piano: PianoSampler; + private readonly noise: NoiseBurstPlayer; + private readonly energy: GardenAudioEnergy; + private readonly gestureState: GardenAudioGestureState; + private readonly pianoEngine: GenerativePianoEngine; + + private currentVibeId: VibeId | null = null; + private currentVibe: VibePreset | null = null; + private lifecycle: AudioLifecycle = 'idle'; + private pianoReleasePhase: PianoReleasePhase = { kind: 'idle' }; + private isMuted = false; + private isGestureActive = false; + private masterVolume: number; + private lastEraserAt = Number.NEGATIVE_INFINITY; + private lastVibeStingerAt = Number.NEGATIVE_INFINITY; + private startRequestId = 0; + private hasLoadedPiano = false; + + public constructor(private readonly config: GardenAudioConfig) { + this.masterVolume = clamp01(config.masterVolume); + this.graph = new GardenAudioGraph(config); + this.piano = new PianoSampler(config, this.graph); + this.noise = new NoiseBurstPlayer(this.graph); + this.energy = new GardenAudioEnergy(config); + this.gestureState = new GardenAudioGestureState(config.input); + this.pianoEngine = new GenerativePianoEngine(config, (note) => this.piano.play(note)); + } + + public start(vibe: VibePreset, options: { userGesture?: boolean } = {}): void { + const isUserGesture = options.userGesture === true; + + if (this.lifecycle === 'destroyed') { + return; + } + + if ( + this.lifecycle === 'started' && + this.currentVibeId === vibe.id && + this.graph.context?.state === 'running' && + this.hasLoadedPiano + ) { + return; + } + + const context = this.graph.ensureContext(isUserGesture); + if (!context) { + return; + } + + const startupRampSeconds = isUserGesture + ? muteRampSeconds + : this.config.fadeInSeconds; + const needsResume = context.state !== 'running' && context.state !== 'closed'; + const startRequestId = ++this.startRequestId; + + if (needsResume) { + if (!isUserGesture) { + return; + } + void context + .resume() + .then(() => { + if (this.graph.context === context && this.lifecycle !== 'destroyed') { + this.completeStart(vibe, { context, startupRampSeconds, startRequestId }); + } + }) + .catch((error) => { + ErrorHandler.addException(error, { + fallbackMessage: 'Could not resume audio playback.', + severity: Severity.WARNING, + }); + }); + return; + } + + this.completeStart(vibe, { context, startupRampSeconds, startRequestId }); + } + + private completeStart( + vibe: VibePreset, + { + context, + startRequestId, + startupRampSeconds, + }: { + context: AudioContext; + startRequestId: number; + startupRampSeconds: number; + } + ): void { + if (this.graph.context !== context || this.lifecycle === 'destroyed') { + return; + } + + if (this.isMuted) { + this.activateMutedStart(vibe, context); + this.graph.setMasterGain(SILENT_AUDIO_GAIN, muteRampSeconds); + return; + } + + void this.piano + .load(context) + .then(() => { + if (!this.canCompleteStart(context, startRequestId)) { + return; + } + + this.activateStart(vibe, context, startupRampSeconds, true); + }) + .catch((error) => { + if (this.canCompleteStart(context, startRequestId)) { + this.activateStart(vibe, context, startupRampSeconds, false); + } + ErrorHandler.addException(error, { + fallbackMessage: 'Could not load piano samples.', + severity: Severity.WARNING, + }); + }); + } + + private canCompleteStart(context: AudioContext, startRequestId: number): boolean { + return ( + this.graph.context === context && + this.lifecycle !== 'destroyed' && + !this.isMuted && + this.startRequestId === startRequestId + ); + } + + private activateStart( + vibe: VibePreset, + context: AudioContext, + startupRampSeconds: number, + cuePiano: boolean + ): void { + this.lifecycle = 'started'; + this.currentVibeId = vibe.id; + this.currentVibe = vibe; + const profile = getVibeProfile(vibe); + this.graph.applyDelayProfile(profile.bpm); + this.graph.setMasterGain(this.masterVolume, startupRampSeconds); + + if (cuePiano) { + this.hasLoadedPiano = true; + this.pianoEngine.cue(context.currentTime, profile); + } + } + + private activateMutedStart(vibe: VibePreset, context: AudioContext): void { + this.lifecycle = 'started'; + this.currentVibeId = vibe.id; + this.currentVibe = vibe; + this.hasLoadedPiano = false; + this.graph.applyDelayProfile(getVibeProfile(vibe).bpm); + if (this.graph.context === context) { + this.pianoEngine.reset(); + } + } + + public changeVibe(vibe: VibePreset, options: { userGesture?: boolean } = {}): void { + const previousVibeId = this.currentVibeId; + this.start(vibe, options); + const didChangeVibe = previousVibeId !== null && previousVibeId !== vibe.id; + + if (didChangeVibe) { + this.piano.stopAll(); + this.hasLoadedPiano = false; + } + + const context = this.graph.context; + if ( + context && + (context.state === 'running' || options.userGesture === true) && + !this.isMuted && + this.lifecycle !== 'destroyed' && + didChangeVibe + ) { + this.playVibeChangeStinger(vibe); + } + } + + public setMuted(isMuted: boolean): void { + if (this.isMuted === isMuted) { + return; + } + + this.isMuted = isMuted; + this.graph.setMasterGain( + isMuted ? SILENT_AUDIO_GAIN : this.masterVolume, + isMuted ? muteRampSeconds : this.config.fadeInSeconds + ); + + if (!isMuted && this.currentVibe && !this.hasLoadedPiano) { + this.start(this.currentVibe); + } + } + + public setMasterVolume(masterVolume: number): void { + this.masterVolume = clamp01(masterVolume); + if (!this.isMuted) { + this.graph.setMasterGain(this.masterVolume, this.config.updateRampSeconds); + } + } + + public beginGesture(): void { + const context = this.graph.context; + if (!context) { + return; + } + + this.isGestureActive = true; + this.pianoReleasePhase = { kind: 'idle' }; + this.graph.setPianoBusGainScale(1, this.config.fadeInSeconds); + this.gestureState.reset(); + this.energy.beginGesture(context.currentTime); + this.pianoEngine.beginGesture(); + } + + public endGesture(): void { + this.gestureState.reset(); + this.isGestureActive = false; + this.pianoReleasePhase = { kind: 'awaiting-fade' }; + this.energy.endGesture(); + this.pianoEngine.endGesture(); + } + + public update(snapshot: GardenAudioSnapshot): void { + const context = this.graph.context; + if (this.lifecycle !== 'started' || !context || this.isMuted) { + return; + } + + this.applyVibe(snapshot.vibe); + const profile = getVibeProfile(snapshot.vibe); + this.energy.update(context.currentTime, profile); + + if (snapshot.isErasing) { + this.energy.silence(); + } + + if (!this.isGestureActive && this.pianoReleasePhase.kind !== 'idle') { + this.updatePianoRelease(snapshot.vibe, context.currentTime); + this.updateDelay(snapshot, profile); + return; + } + + this.pianoEngine.renderLookahead({ + vibe: snapshot.vibe, + now: context.currentTime, + activity: snapshot.isErasing + ? this.config.eraser.pianoActivity + : this.energy.getLevel(), + }); + this.updateDelay(snapshot, profile); + } + + public stroke(stroke: GardenAudioStroke): void { + if (this.lifecycle !== 'started' || this.isMuted) { + return; + } + + const context = this.graph.context; + if (!context) { + return; + } + if (!this.isGestureActive) { + return; + } + + const metrics = getStrokeMetrics(stroke); + const now = context.currentTime; + + const frame = this.gestureState.recordStroke({ metrics }); + const strokeEnergy = frame.activity; + + if (stroke.isErasing) { + this.playEraser(strokeEnergy, now); + return; + } + + const profile = getVibeProfile(stroke.vibe); + this.energy.recordStroke(strokeEnergy, profile); + this.pianoEngine.recordStroke({ + vibe: stroke.vibe, + now, + activity: strokeEnergy, + maniaAmount: frame.maniaAmount, + }); + } + + public async destroy(): Promise { + this.lifecycle = 'destroyed'; + await this.graph.close(); + + this.piano.reset(); + this.hasLoadedPiano = false; + this.energy.reset(); + this.gestureState.reset(); + this.pianoEngine.reset(); + this.currentVibeId = null; + this.currentVibe = null; + this.isGestureActive = false; + this.pianoReleasePhase = { kind: 'idle' }; + this.lastEraserAt = Number.NEGATIVE_INFINITY; + this.lastVibeStingerAt = Number.NEGATIVE_INFINITY; + } + + private playVibeChangeStinger(vibe: VibePreset): void { + const context = this.graph.context; + if (!context) { + return; + } + + const now = context.currentTime; + if (now - this.lastVibeStingerAt < vibeChangeStingerMinIntervalSeconds) { + return; + } + + this.lastVibeStingerAt = now; + this.pianoEngine.playVibeChangeStinger(vibe, now); + } + + private updatePianoRelease(vibe: VibePreset, now: number): void { + if (this.pianoReleasePhase.kind === 'awaiting-fade') { + const fadeAt = this.pianoEngine.release(vibe, now); + if (now < fadeAt) { + this.pianoReleasePhase = { kind: 'scheduled-fade', fadeAt }; + return; + } + + this.graph.setPianoBusGainScale(0, brushUpPianoBusFadeSeconds); + this.pianoReleasePhase = { + kind: 'settling', + stopAt: now + brushUpPianoBusFadeSettleSeconds, + }; + return; + } + + if ( + this.pianoReleasePhase.kind === 'scheduled-fade' && + now >= this.pianoReleasePhase.fadeAt + ) { + this.graph.setPianoBusGainScale(0, brushUpPianoBusFadeSeconds); + this.pianoReleasePhase = { + kind: 'settling', + stopAt: now + brushUpPianoBusFadeSettleSeconds, + }; + return; + } + + if ( + this.pianoReleasePhase.kind === 'settling' && + now >= this.pianoReleasePhase.stopAt + ) { + this.piano.stopAll(); + this.pianoEngine.reset(); + this.hasLoadedPiano = false; + this.pianoReleasePhase = { kind: 'idle' }; + } + } + + private playEraser(activity: number, now: number): void { + if (!this.graph.context) { + return; + } + + const distanceActivity = clamp01(activity); + if (distanceActivity <= 0) { + return; + } + + const filterHz = + this.config.eraser.filterMinHz + + (this.config.eraser.filterMaxHz - this.config.eraser.filterMinHz) * + distanceActivity; + + if (now - this.lastEraserAt >= this.config.eraser.minIntervalSeconds) { + this.lastEraserAt = now; + this.noise.play({ + startTime: now, + durationSeconds: this.config.eraser.durationSeconds, + gain: this.config.eraser.noiseGain * distanceActivity, + filterHz, + pan: this.config.eraser.pan, + }); + } + } + + private updateDelay( + snapshot: GardenAudioSnapshot, + profile: GardenAudioVibeProfile + ): void { + const context = this.graph.context; + if (!context) { + return; + } + + const activity = snapshot.isErasing + ? this.config.delay.erasingActivity + : this.energy.getLevel(); + this.graph.updateDelay(activity, profile.bpm); + } + + private applyVibe(vibe: VibePreset): void { + if (!this.graph.context || this.currentVibeId === vibe.id) { + return; + } + + this.currentVibeId = vibe.id; + this.currentVibe = vibe; + const profile = getVibeProfile(vibe); + this.graph.applyDelayProfile(profile.bpm); + this.pianoEngine.cue(this.graph.context.currentTime, profile); + this.hasLoadedPiano = true; + } +} diff --git a/src/audio/generative-piano-tuning.ts b/src/audio/generative-piano-tuning.ts new file mode 100644 index 0000000..e76d6f4 --- /dev/null +++ b/src/audio/generative-piano-tuning.ts @@ -0,0 +1,443 @@ +export interface GardenAudioRegister { + midiMin: number; + midiMax: number; + preferredMidi: number; + pan: number; +} + +export interface GardenAudioStylePool extends GardenAudioRegister { + scaleDegrees: Array; +} + +interface GardenAudioStyleVoice { + scaleDegreeOffset: number; + velocityMultiplier: number; + panOffset: number; +} + +interface GenerativePianoTuning { + stylePools: [GardenAudioStylePool, GardenAudioStylePool, GardenAudioStylePool]; + padRegisters: [GardenAudioRegister, GardenAudioRegister, GardenAudioRegister]; + vibeChangeStinger: { + velocities: [number, number, number]; + pans: [number, number, number]; + delaySends: [number, number, number]; + lowpassExpression: number; + noteDurationSeconds: number; + spacingSeconds: number; + }; + releaseResolution: { + durationSeconds: number; + fadeAfterSeconds: number; + velocities: [number, number, number]; + delaySend: number; + lowpassExpression: number; + strumSeconds: number; + }; + highActivityExtra: { + barOffset: number; + expressionMultiplier: number; + }; + padChord: { + velocities: [number, number, number]; + expressionVelocityWeight: number; + delaySend: number; + lowpassExpressionWeight: number; + }; + supportNote: { + velocityBase: number; + velocityExpressionWeight: number; + durationBaseSeconds: number; + durationExpressionSeconds: number; + delaySendBase: number; + delaySendExpressionWeight: number; + lowpassExpressionWeight: number; + expressionThreshold: number; + offsetsByStyle: [Array, Array, Array]; + }; + textureNote: { + velocityBase: number; + velocityExpressionWeight: number; + durationBaseSeconds: number; + durationExpressionSeconds: number; + delaySendBase: number; + delaySendExpressionWeight: number; + idleExpressionThreshold: number; + mediumExpressionThreshold: number; + intenseSpacing: number; + idlePhase: number; + }; + gestureAccent: { + rotationStrengthMultiplier: number; + quantizeStepLookahead: number; + velocityBase: number; + velocityStrengthWeight: number; + durationBaseSeconds: number; + durationStrengthSeconds: number; + delaySend: number; + }; + touchNote: { + velocityBase: number; + velocityStrengthWeight: number; + durationBaseSeconds: number; + durationStrengthSeconds: number; + delaySend: number; + lowpassBaseExpression: number; + lowpassStrengthWeight: number; + }; + brushPhrase: { + initialMotifOffset: number; + energyDecaySeconds: number; + maniaDecaySeconds: number; + layerIntensityBase: number; + layerIntensityManiaWeight: number; + frameActivityWeight: number; + frameManiaWeight: number; + }; + brushStream: { + inferredManiaThreshold: number; + inferredManiaRange: number; + registerManiaShift: number; + chordToneEverySteps: number; + durationBaseSeconds: number; + durationIntensitySeconds: number; + durationManiaSeconds: number; + durationMinSeconds: number; + durationMaxSeconds: number; + delaySendBase: number; + delaySendIntensityWeight: number; + delaySendManiaWeight: number; + delaySendMin: number; + delaySendMax: number; + velocityBase: number; + velocityIntensityWeight: number; + lowpassBaseExpression: number; + lowpassIntensityWeight: number; + lowpassManiaWeight: number; + intenseThreshold: number; + activeThreshold: number; + }; + brushStreamEcho: { + maniaThreshold: number; + stepModulo: number; + stepRemainder: number; + intensityThreshold: number; + octaveSemitones: number; + maxMidi: number; + velocityBase: number; + velocityIntensityWeight: number; + durationMinSeconds: number; + durationScale: number; + panScale: number; + delaySendMin: number; + delaySendScale: number; + lowpassBaseExpression: number; + lowpassManiaWeight: number; + }; + brushMotif: { + highThreshold: number; + mediumThreshold: number; + highOffset: number; + mediumOffset: number; + lowOffset: number; + }; + registerBias: { + maniaShiftSemitones: number; + midiMin: number; + midiMaxForMin: number; + minimumSpan: number; + midiMax: number; + }; + candidateOctaveSearch: { + min: number; + max: number; + }; + stereoWidth: { + idle: number; + active: number; + intense: number; + intenseThreshold: number; + }; + stylePanOffsetScale: number; + lowpass: { + midiBase: number; + midiRange: number; + midiLiftHz: number; + expressionBase: number; + expressionWeight: number; + }; + styleRotationBars: number; + chordBars: number; + supportBarSpacing: number; + supportBarOffset: number; + idleTextureBarSpacing: number; + mediumTextureBarSpacing: number; + textureBeat: number; + highActivityExtraBeat: number; + highActivityExtraThreshold: number; + noteScorePreferenceWeight: number; + noteScoreRegisterWeight: number; + noteScoreChordToneWeight: number; + noteScoreRepeatPenalty: number; + gestureAccentMinIntervalSeconds: number; + strokeAccentMinSteps: number; + strokeAccentThreshold: number; + maxBrushPhraseLayers: number; + maxBrushStreamNotesPerBar: number; + brushLayerBaseSeconds: number; + brushLayerEnergySeconds: number; + brushLayerMinIntensity: number; + brushStreamIdleIntervalBeats: number; + brushStreamActiveIntervalBeats: number; + brushStreamIntenseIntervalBeats: number; + brushMotifMaxSteps: number; + brushMotifCanonDelaySeconds: number; + padDurationBarScale: number; +} + +export const generativePianoTuning: GenerativePianoTuning = { + stylePools: [ + { + midiMin: 48, + midiMax: 67, + preferredMidi: 55, + pan: -0.18, + scaleDegrees: [0, 1, 2, 4], + }, + { + midiMin: 55, + midiMax: 74, + preferredMidi: 63, + pan: 0, + scaleDegrees: [1, 2, 3, 5], + }, + { + midiMin: 62, + midiMax: 78, + preferredMidi: 70, + pan: 0.18, + scaleDegrees: [2, 3, 4, 6], + }, + ], + padRegisters: [ + { + midiMin: 40, + midiMax: 55, + preferredMidi: 48, + pan: -0.12, + }, + { + midiMin: 48, + midiMax: 64, + preferredMidi: 55, + pan: 0.08, + }, + { + midiMin: 58, + midiMax: 76, + preferredMidi: 67, + pan: 0.2, + }, + ], + vibeChangeStinger: { + velocities: [0.1, 0.085, 0.07], + pans: [-0.16, 0, 0.16], + delaySends: [0.012, 0.014, 0.016], + lowpassExpression: 0.35, + noteDurationSeconds: 1.1, + spacingSeconds: 0.08, + }, + releaseResolution: { + durationSeconds: 3.4, + fadeAfterSeconds: 2.4, + velocities: [0.064, 0.05, 0.038], + delaySend: 0.018, + lowpassExpression: 0.34, + strumSeconds: 0.055, + }, + highActivityExtra: { + barOffset: 1, + expressionMultiplier: 0.9, + }, + padChord: { + velocities: [0.046, 0.036, 0.029], + expressionVelocityWeight: 0.018, + delaySend: 0.008, + lowpassExpressionWeight: 0.24, + }, + supportNote: { + velocityBase: 0.105, + velocityExpressionWeight: 0.07, + durationBaseSeconds: 1.35, + durationExpressionSeconds: 0.4, + delaySendBase: 0.016, + delaySendExpressionWeight: 0.006, + lowpassExpressionWeight: 0.7, + expressionThreshold: 0.55, + offsetsByStyle: [ + [0, 2, 12], + [1, 2, 0, 12], + [2, 12, 3, 13], + ], + }, + textureNote: { + velocityBase: 0.09, + velocityExpressionWeight: 0.08, + durationBaseSeconds: 0.62, + durationExpressionSeconds: 0.24, + delaySendBase: 0.016, + delaySendExpressionWeight: 0.006, + idleExpressionThreshold: 0.35, + mediumExpressionThreshold: 0.7, + intenseSpacing: 1, + idlePhase: 1, + }, + gestureAccent: { + rotationStrengthMultiplier: 3, + quantizeStepLookahead: 1, + velocityBase: 0.12, + velocityStrengthWeight: 0.09, + durationBaseSeconds: 0.48, + durationStrengthSeconds: 0.22, + delaySend: 0.012, + }, + touchNote: { + velocityBase: 0.14, + velocityStrengthWeight: 0.11, + durationBaseSeconds: 0.55, + durationStrengthSeconds: 0.18, + delaySend: 0.006, + lowpassBaseExpression: 0.55, + lowpassStrengthWeight: 0.35, + }, + brushPhrase: { + initialMotifOffset: -1, + energyDecaySeconds: 0.72, + maniaDecaySeconds: 0.54, + layerIntensityBase: 0.8, + layerIntensityManiaWeight: 0.42, + frameActivityWeight: 0.42, + frameManiaWeight: 0.18, + }, + brushStream: { + inferredManiaThreshold: 0.82, + inferredManiaRange: 0.18, + registerManiaShift: 0.3, + chordToneEverySteps: 4, + durationBaseSeconds: 0.48, + durationIntensitySeconds: 0.08, + durationManiaSeconds: 0.34, + durationMinSeconds: 0.14, + durationMaxSeconds: 0.62, + delaySendBase: 0.012, + delaySendIntensityWeight: 0.011, + delaySendManiaWeight: 0.006, + delaySendMin: 0.006, + delaySendMax: 0.032, + velocityBase: 0.1, + velocityIntensityWeight: 0.1, + lowpassBaseExpression: 0.39, + lowpassIntensityWeight: 0.48, + lowpassManiaWeight: 0.18, + intenseThreshold: 0.68, + activeThreshold: 0.34, + }, + brushStreamEcho: { + maniaThreshold: 0.92, + stepModulo: 3, + stepRemainder: 1, + intensityThreshold: 0.95, + octaveSemitones: 12, + maxMidi: 84, + velocityBase: 0.035, + velocityIntensityWeight: 0.04, + durationMinSeconds: 0.11, + durationScale: 0.68, + panScale: -0.75, + delaySendMin: 0.006, + delaySendScale: 0.72, + lowpassBaseExpression: 0.62, + lowpassManiaWeight: 0.24, + }, + brushMotif: { + highThreshold: 0.82, + mediumThreshold: 0.55, + highOffset: 1, + mediumOffset: 0, + lowOffset: -1, + }, + registerBias: { + maniaShiftSemitones: 2, + midiMin: 36, + midiMaxForMin: 86, + minimumSpan: 4, + midiMax: 91, + }, + candidateOctaveSearch: { + min: -3, + max: 3, + }, + stereoWidth: { + idle: 0.46, + active: 0.9, + intense: 1.16, + intenseThreshold: 0.72, + }, + stylePanOffsetScale: 0.35, + lowpass: { + midiBase: 48, + midiRange: 33, + midiLiftHz: 500, + expressionBase: 0.58, + expressionWeight: 0.32, + }, + styleRotationBars: 2, + chordBars: 4, + supportBarSpacing: 2, + supportBarOffset: 1, + idleTextureBarSpacing: 2, + mediumTextureBarSpacing: 1, + textureBeat: 2, + highActivityExtraBeat: 3, + highActivityExtraThreshold: 0.45, + noteScorePreferenceWeight: 1.8, + noteScoreRegisterWeight: 0.28, + noteScoreChordToneWeight: 0.75, + noteScoreRepeatPenalty: 3.2, + gestureAccentMinIntervalSeconds: 2.5, + strokeAccentMinSteps: 12, + strokeAccentThreshold: 0.58, + maxBrushPhraseLayers: 3, + maxBrushStreamNotesPerBar: 7, + brushLayerBaseSeconds: 5.5, + brushLayerEnergySeconds: 2.5, + brushLayerMinIntensity: 0.12, + brushStreamIdleIntervalBeats: 2, + brushStreamActiveIntervalBeats: 1, + brushStreamIntenseIntervalBeats: 0.75, + brushMotifMaxSteps: 8, + brushMotifCanonDelaySeconds: 0.055, + padDurationBarScale: 0.82, +}; + +export const styleVoices: [ + GardenAudioStyleVoice, + GardenAudioStyleVoice, + GardenAudioStyleVoice, +] = [ + { + scaleDegreeOffset: 0, + velocityMultiplier: 0.92, + panOffset: -0.14, + }, + { + scaleDegreeOffset: 1, + velocityMultiplier: 1, + panOffset: 0, + }, + { + scaleDegreeOffset: 2, + velocityMultiplier: 0.86, + panOffset: 0.14, + }, +]; diff --git a/src/audio/generative-piano.ts b/src/audio/generative-piano.ts new file mode 100644 index 0000000..fbe1201 --- /dev/null +++ b/src/audio/generative-piano.ts @@ -0,0 +1,1349 @@ +import { clamp, clamp01 } from '../utils/math'; +import type { VibePreset } from '../vibes'; +import type { + GardenAudioChord, + GardenAudioConfig, + GardenAudioVibeProfile, +} from './garden-audio-config'; +import { getVibeProfile, PITCH_SEMITONES_PER_OCTAVE } from './garden-audio-music'; +import type { PianoNote } from './garden-audio-types'; +import { + generativePianoTuning, + styleVoices, + type GardenAudioRegister, + type GardenAudioStylePool, +} from './generative-piano-tuning'; +import { PIANO_SCHEDULE_AHEAD_SECONDS } from './piano-sampler'; + +const GENERATIVE_LOOKAHEAD_SECONDS = 0.3; +const GENERATIVE_START_DELAY_SECONDS = 0.02; +const TEXTURE_ONSET_EXPRESSION = 0.15; +const SUPPORT_ONSET_EXPRESSION = 0.4; + +const chordVoicings: Record< + GardenAudioChord['quality'], + { closed: Array; open: Array } +> = { + major: { + closed: [0, 4, 7, 12, 16], + open: [0, 7, 12, 16], + }, + minor: { + closed: [0, 3, 7, 12, 15], + open: [0, 7, 12, 15], + }, + sus2: { + closed: [0, 2, 7, 12, 14], + open: [0, 7, 12, 14], + }, + sus4: { + closed: [0, 5, 7, 12, 17], + open: [0, 7, 12, 17], + }, +}; + +const getChordIntervals = ( + chord: GardenAudioChord, + openVoicing: boolean +): Array => { + const voicing = chordVoicings[chord.quality]; + return openVoicing ? voicing.open : voicing.closed; +}; + +const degreeToSemitone = (profile: GardenAudioVibeProfile, degree: number): number => { + const scaleIndex = + ((degree % profile.scale.length) + profile.scale.length) % profile.scale.length; + const octave = Math.floor(degree / profile.scale.length); + return profile.scale[scaleIndex] + octave * PITCH_SEMITONES_PER_OCTAVE; +}; + +type GardenAudioStyleIndex = 0 | 1 | 2; + +interface PitchCandidate { + midi: number; + preference: number; + chordToneDistance: number; +} + +interface PitchSource { + baseMidi: number; + offsets: ReadonlyArray; + chordOffsets?: ReadonlyArray; +} + +interface BrushPhraseLayer { + vibe: VibePreset; + startedAt: number; + lastUpdatedAt: number; + expiresAt: number; + styleIndex: GardenAudioStyleIndex; + energy: number; + motifOffsets: Array; + maniaAmount: number; +} + +export class GenerativePianoEngine { + private nextBeatStep: number | null = null; + private timelineStartedAt: number | null = null; + private activeProfile: GardenAudioVibeProfile | null = null; + private isWaitingForGestureAccent = false; + private lastGestureAccentAt = Number.NEGATIVE_INFINITY; + private lastStrokeAccentStep = Number.NEGATIVE_INFINITY; + private readonly lastMidiByStyle: [number | null, number | null, number | null] = [ + null, + null, + null, + ]; + private readonly lastPadMidiByVoice: [number | null, number | null, number | null] = [ + null, + null, + null, + ]; + private brushPhraseLayers: Array = []; + private nextBrushStreamStep: number | null = null; + private brushStreamNoteIndex = 0; + private lastBrushStreamMidi: number | null = null; + private readonly brushStreamNoteCountsByBar = new Map(); + + public constructor( + private readonly config: GardenAudioConfig, + private readonly playNote: (note: PianoNote) => void + ) {} + + public prime(now: number, profile: GardenAudioVibeProfile): void { + this.activeProfile = profile; + this.timelineStartedAt ??= now; + this.nextBeatStep ??= 0; + this.nextBrushStreamStep ??= 0; + } + + public cue(now: number, profile?: GardenAudioVibeProfile): void { + if (profile) { + this.activeProfile = profile; + } + this.nextBeatStep = 0; + this.timelineStartedAt = now; + this.nextBrushStreamStep = 0; + this.brushStreamNoteIndex = 0; + this.lastBrushStreamMidi = null; + this.brushStreamNoteCountsByBar.clear(); + } + + public beginGesture(): void { + this.isWaitingForGestureAccent = true; + } + + public endGesture(): void { + this.isWaitingForGestureAccent = false; + } + + public release(vibe: VibePreset, now: number): number { + const profile = getVibeProfile(vibe); + this.prime(now, profile); + this.isWaitingForGestureAccent = false; + const releaseStep = this.getReleaseResolutionStep(now); + const releaseStart = Math.max(now, this.getTimeForStep(releaseStep)); + + this.playReleaseResolution(profile, releaseStep, releaseStart); + this.nextBeatStep = null; + this.nextBrushStreamStep = null; + this.brushPhraseLayers = []; + this.brushStreamNoteCountsByBar.clear(); + + return releaseStart + generativePianoTuning.releaseResolution.fadeAfterSeconds; + } + + private recordTouchDown({ + vibe, + now, + strength, + maniaAmount = 0, + }: { + vibe: VibePreset; + now: number; + strength: number; + maniaAmount?: number; + }): void { + const normalizedStrength = clamp01(strength); + const normalizedManiaAmount = clamp01(maniaAmount); + const styleIndex = this.getStyleIndex(now); + + this.isWaitingForGestureAccent = false; + this.lastGestureAccentAt = now; + this.lastStrokeAccentStep = this.getStepIndexAtTime(now); + this.startBrushPhraseLayer({ + vibe, + now, + strength: normalizedStrength, + styleIndex, + maniaAmount: normalizedManiaAmount, + }); + this.playTouchNote({ + vibe, + now, + styleIndex, + strength: normalizedStrength, + }); + } + + public recordStroke({ + vibe, + now, + activity, + maniaAmount = 0, + }: { + vibe: VibePreset; + now: number; + activity: number; + maniaAmount?: number; + }): void { + const profile = getVibeProfile(vibe); + this.prime(now, profile); + const strength = clamp01(activity); + const normalizedManiaAmount = clamp01(maniaAmount); + const styleIndex = this.getStyleIndex(now); + const accentStep = this.getNextStepIndexAt( + now, + generativePianoTuning.gestureAccent.quantizeStepLookahead + ); + + if ( + this.isWaitingForGestureAccent && + now - this.lastGestureAccentAt >= + generativePianoTuning.gestureAccentMinIntervalSeconds + ) { + this.recordTouchDown({ + vibe, + now, + strength, + maniaAmount: normalizedManiaAmount, + }); + return; + } + + this.isWaitingForGestureAccent = false; + this.updateBrushPhraseLayer({ + now, + strength, + styleIndex, + maniaAmount: normalizedManiaAmount, + }); + if ( + strength >= generativePianoTuning.strokeAccentThreshold && + accentStep - this.lastStrokeAccentStep >= generativePianoTuning.strokeAccentMinSteps + ) { + this.lastStrokeAccentStep = accentStep; + this.playGestureAccent(vibe, accentStep, styleIndex, strength); + } + } + + public renderLookahead({ + vibe, + now, + activity, + lookaheadSeconds = GENERATIVE_LOOKAHEAD_SECONDS, + }: { + vibe: VibePreset; + now: number; + activity: number; + lookaheadSeconds?: number; + }): void { + const profile = getVibeProfile(vibe); + this.prime(now, profile); + this.skipLateBeats(now); + + if (this.nextBeatStep === null) { + return; + } + + const lookaheadEnd = now + lookaheadSeconds; + const expression = this.getExpression(activity); + while (this.getTimeForStep(this.nextBeatStep) <= lookaheadEnd) { + const beatIndex = this.getBeatIndexForStep(this.nextBeatStep); + this.renderBeat({ + profile, + beatIndex, + startTime: this.getTimeForStep(this.nextBeatStep), + expression, + }); + this.nextBeatStep += this.config.rhythm.stepsPerBeat; + } + this.renderBrushPhraseLayers({ + vibe, + now, + lookaheadEnd, + activity: expression, + }); + } + + public playVibeChangeStinger(vibe: VibePreset, now: number): void { + const profile = getVibeProfile(vibe); + const chord = this.getChord(profile, 0); + const intervals = getChordIntervals(chord, true); + const rootMidi = profile.rootMidi + chord.rootOffset; + const stinger = generativePianoTuning.vibeChangeStinger; + const offsetsByVoice: ReadonlyArray> = [ + [0], + [intervals[1], intervals[2]], + [intervals[3], intervals[2]], + ]; + + offsetsByVoice.forEach((offsets, index) => { + const midi = this.chooseMidi( + { baseMidi: rootMidi, offsets }, + generativePianoTuning.padRegisters[index] + ); + this.playProfileNote(profile, { + midi, + velocity: stinger.velocities[index], + pan: stinger.pans[index], + delaySend: stinger.delaySends[index], + durationSeconds: stinger.noteDurationSeconds, + role: 'stinger', + lowpassHz: this.getLowpassHz(profile, midi, stinger.lowpassExpression), + startTime: now + index * stinger.spacingSeconds, + }); + }); + } + + public reset(): void { + this.nextBeatStep = null; + this.timelineStartedAt = null; + this.activeProfile = null; + this.isWaitingForGestureAccent = false; + this.lastGestureAccentAt = Number.NEGATIVE_INFINITY; + this.lastStrokeAccentStep = Number.NEGATIVE_INFINITY; + this.lastMidiByStyle.fill(null); + this.lastPadMidiByVoice.fill(null); + this.brushPhraseLayers = []; + this.nextBrushStreamStep = null; + this.brushStreamNoteIndex = 0; + this.lastBrushStreamMidi = null; + this.brushStreamNoteCountsByBar.clear(); + } + + private playProfileNote(profile: GardenAudioVibeProfile, note: PianoNote): void { + this.playNote({ + ...note, + sustainSeconds: profile.noteLength, + }); + } + + private renderBeat({ + profile, + beatIndex, + startTime, + expression, + }: { + profile: GardenAudioVibeProfile; + beatIndex: number; + startTime: number; + expression: number; + }): void { + const beatsPerBar = this.getBeatsPerBar(); + const beatInBar = beatIndex % beatsPerBar; + const barIndex = Math.floor(beatIndex / beatsPerBar); + const styleIndex = this.getStyleIndex(startTime); + + if (beatInBar === 0 && barIndex % generativePianoTuning.chordBars === 0) { + this.playPadChord(profile, barIndex, startTime, expression); + } + + if (beatInBar === 0 && this.shouldPlaySupport(expression, barIndex)) { + this.playSupportNote(profile, barIndex, startTime, expression, styleIndex); + } + + if ( + beatInBar === generativePianoTuning.textureBeat && + this.shouldPlayTexture(expression, barIndex) + ) { + this.playTextureNote(profile, barIndex, startTime, expression, styleIndex); + } + + if ( + beatInBar === generativePianoTuning.highActivityExtraBeat && + expression >= generativePianoTuning.highActivityExtraThreshold + ) { + this.playTextureNote( + profile, + barIndex + generativePianoTuning.highActivityExtra.barOffset, + startTime, + expression * generativePianoTuning.highActivityExtra.expressionMultiplier, + styleIndex + ); + } + } + + private playPadChord( + profile: GardenAudioVibeProfile, + barIndex: number, + startTime: number, + expression: number + ): void { + const chord = this.getChord(profile, barIndex); + const intervals = getChordIntervals(chord, true); + const rootMidi = profile.rootMidi + chord.rootOffset; + const durationSeconds = + this.getBarDurationSeconds() * + generativePianoTuning.chordBars * + generativePianoTuning.padDurationBarScale; + const notes = [ + { + source: { baseMidi: rootMidi, offsets: [0] }, + register: generativePianoTuning.padRegisters[0], + velocity: generativePianoTuning.padChord.velocities[0], + }, + { + source: { baseMidi: rootMidi, offsets: [intervals[1]] }, + register: generativePianoTuning.padRegisters[1], + velocity: generativePianoTuning.padChord.velocities[1], + }, + { + source: { baseMidi: rootMidi, offsets: [intervals[3], intervals[2]] }, + register: generativePianoTuning.padRegisters[2], + velocity: generativePianoTuning.padChord.velocities[2], + }, + ]; + + notes.forEach(({ source, register, velocity }, index) => { + const midi = this.chooseMidi( + source, + register, + this.lastPadMidiByVoice[index], + false + ); + this.lastPadMidiByVoice[index] = midi; + this.playProfileNote(profile, { + midi, + velocity: + velocity + expression * generativePianoTuning.padChord.expressionVelocityWeight, + startTime, + durationSeconds, + pan: this.getActivityPan(register.pan, expression), + role: 'pad', + delaySend: generativePianoTuning.padChord.delaySend, + lowpassHz: this.getLowpassHz( + profile, + midi, + expression * generativePianoTuning.padChord.lowpassExpressionWeight + ), + }); + }); + } + + private playReleaseResolution( + profile: GardenAudioVibeProfile, + stepIndex: number, + startTime: number + ): void { + const chord = this.getChord(profile, this.getBarIndexForStep(stepIndex)); + const intervals = getChordIntervals(chord, true); + const rootMidi = profile.rootMidi + chord.rootOffset; + const release = generativePianoTuning.releaseResolution; + const offsetsByVoice: ReadonlyArray> = [ + [0], + [intervals[1], intervals[2]], + [intervals[3], intervals[2]], + ]; + + offsetsByVoice.forEach((offsets, index) => { + const register = generativePianoTuning.padRegisters[index]; + const midi = this.chooseMidi( + { baseMidi: rootMidi, offsets }, + register, + null, + false + ); + this.playProfileNote(profile, { + midi, + velocity: release.velocities[index], + startTime: startTime + index * release.strumSeconds, + durationSeconds: release.durationSeconds, + pan: this.getActivityPan(register.pan, 0), + role: 'pad', + delaySend: release.delaySend, + lowpassHz: this.getLowpassHz(profile, midi, release.lowpassExpression), + }); + }); + } + + private playSupportNote( + profile: GardenAudioVibeProfile, + barIndex: number, + startTime: number, + expression: number, + styleIndex: GardenAudioStyleIndex + ): void { + const pool = generativePianoTuning.stylePools[styleIndex]; + const chord = this.getChord(profile, barIndex); + const chordIntervals = getChordIntervals(chord, false); + const rootMidi = profile.rootMidi + chord.rootOffset; + const midi = this.chooseMidi( + { + baseMidi: rootMidi, + offsets: this.getSupportOffsets(chordIntervals, styleIndex), + chordOffsets: chordIntervals, + }, + pool, + this.lastMidiByStyle[styleIndex], + true + ); + + this.lastMidiByStyle[styleIndex] = midi; + this.playProfileNote(profile, { + midi, + velocity: + (generativePianoTuning.supportNote.velocityBase + + expression * generativePianoTuning.supportNote.velocityExpressionWeight) * + styleVoices[styleIndex].velocityMultiplier, + startTime, + durationSeconds: + generativePianoTuning.supportNote.durationBaseSeconds + + expression * generativePianoTuning.supportNote.durationExpressionSeconds, + pan: this.getStylePan(styleIndex, expression), + role: 'support', + delaySend: + generativePianoTuning.supportNote.delaySendBase + + expression * generativePianoTuning.supportNote.delaySendExpressionWeight, + lowpassHz: this.getLowpassHz( + profile, + midi, + expression * generativePianoTuning.supportNote.lowpassExpressionWeight + ), + }); + } + + private playTextureNote( + profile: GardenAudioVibeProfile, + barIndex: number, + startTime: number, + expression: number, + styleIndex: GardenAudioStyleIndex + ): void { + const pool = generativePianoTuning.stylePools[styleIndex]; + const chord = this.getChord(profile, barIndex); + const chordIntervals = getChordIntervals(chord, false); + const degrees = this.rotate(pool.scaleDegrees, barIndex + styleIndex); + const midi = this.chooseMidi( + { + baseMidi: profile.rootMidi, + offsets: degrees.map((degree) => degreeToSemitone(profile, degree)), + chordOffsets: this.getChordOffsets(chord, chordIntervals), + }, + pool, + this.lastMidiByStyle[styleIndex], + true + ); + + this.lastMidiByStyle[styleIndex] = midi; + this.playProfileNote(profile, { + midi, + velocity: + (generativePianoTuning.textureNote.velocityBase + + expression * generativePianoTuning.textureNote.velocityExpressionWeight) * + styleVoices[styleIndex].velocityMultiplier, + startTime, + durationSeconds: + generativePianoTuning.textureNote.durationBaseSeconds + + expression * generativePianoTuning.textureNote.durationExpressionSeconds, + pan: this.getStylePan(styleIndex, expression), + role: 'texture', + delaySend: + generativePianoTuning.textureNote.delaySendBase + + expression * generativePianoTuning.textureNote.delaySendExpressionWeight, + lowpassHz: this.getLowpassHz(profile, midi, expression), + }); + } + + private playGestureAccent( + vibe: VibePreset, + stepIndex: number, + styleIndex: GardenAudioStyleIndex, + strength: number + ): void { + const profile = getVibeProfile(vibe); + const pool = generativePianoTuning.stylePools[styleIndex]; + const startTime = this.getTimeForStep(stepIndex); + const chord = this.getChord(profile, this.getBarIndexForStep(stepIndex)); + const chordIntervals = getChordIntervals(chord, false); + const degrees = this.rotate( + pool.scaleDegrees, + Math.round( + strength * generativePianoTuning.gestureAccent.rotationStrengthMultiplier + ) + ); + + const midi = this.chooseMidi( + { + baseMidi: profile.rootMidi, + offsets: degrees.map((degree) => degreeToSemitone(profile, degree)), + chordOffsets: this.getChordOffsets(chord, chordIntervals), + }, + pool, + this.lastMidiByStyle[styleIndex], + true + ); + + this.lastMidiByStyle[styleIndex] = midi; + this.playProfileNote(profile, { + midi, + velocity: + (generativePianoTuning.gestureAccent.velocityBase + + strength * generativePianoTuning.gestureAccent.velocityStrengthWeight) * + styleVoices[styleIndex].velocityMultiplier, + startTime, + durationSeconds: + generativePianoTuning.gestureAccent.durationBaseSeconds + + strength * generativePianoTuning.gestureAccent.durationStrengthSeconds, + pan: this.getStylePan(styleIndex, strength), + role: 'gesture', + delaySend: generativePianoTuning.gestureAccent.delaySend, + lowpassHz: this.getLowpassHz(profile, midi, strength), + }); + } + + private playTouchNote({ + vibe, + now, + styleIndex, + strength, + }: { + vibe: VibePreset; + now: number; + styleIndex: GardenAudioStyleIndex; + strength: number; + }): void { + const profile = getVibeProfile(vibe); + const pool = generativePianoTuning.stylePools[styleIndex]; + const chord = this.getChord(profile, this.getGlobalBarIndex(now)); + const chordIntervals = getChordIntervals(chord, false); + const rootMidi = profile.rootMidi + chord.rootOffset; + const midi = this.chooseMidi( + { + baseMidi: rootMidi, + offsets: this.getSupportOffsets(chordIntervals, styleIndex), + }, + pool, + this.lastMidiByStyle[styleIndex], + true + ); + + this.lastMidiByStyle[styleIndex] = midi; + this.lastBrushStreamMidi = midi; + this.playProfileNote(profile, { + midi, + velocity: + (generativePianoTuning.touchNote.velocityBase + + strength * generativePianoTuning.touchNote.velocityStrengthWeight) * + styleVoices[styleIndex].velocityMultiplier, + startTime: now, + durationSeconds: + generativePianoTuning.touchNote.durationBaseSeconds + + strength * generativePianoTuning.touchNote.durationStrengthSeconds, + pan: this.getStylePan(styleIndex, strength), + role: 'gesture', + delaySend: generativePianoTuning.touchNote.delaySend, + lowpassHz: this.getLowpassHz( + profile, + midi, + clamp01( + generativePianoTuning.touchNote.lowpassBaseExpression + + strength * generativePianoTuning.touchNote.lowpassStrengthWeight + ) + ), + }); + } + + private startBrushPhraseLayer({ + vibe, + now, + strength, + styleIndex, + maniaAmount, + }: { + vibe: VibePreset; + now: number; + strength: number; + styleIndex: GardenAudioStyleIndex; + maniaAmount: number; + }): void { + const lifetimeSeconds = + generativePianoTuning.brushLayerBaseSeconds + + strength * generativePianoTuning.brushLayerEnergySeconds; + const expiresAt = this.getNextBarTimeAt(now + lifetimeSeconds); + + this.brushPhraseLayers.push({ + vibe, + startedAt: now, + lastUpdatedAt: now, + expiresAt, + styleIndex, + energy: strength, + motifOffsets: [styleIndex + generativePianoTuning.brushPhrase.initialMotifOffset], + maniaAmount, + }); + + if (this.brushPhraseLayers.length > generativePianoTuning.maxBrushPhraseLayers) { + this.brushPhraseLayers = this.brushPhraseLayers.slice( + -generativePianoTuning.maxBrushPhraseLayers + ); + } + } + + private updateBrushPhraseLayer({ + now, + strength, + styleIndex, + maniaAmount, + }: { + now: number; + strength: number; + styleIndex: GardenAudioStyleIndex; + maniaAmount: number; + }): void { + const layer = this.brushPhraseLayers[this.brushPhraseLayers.length - 1]; + if (!layer || layer.expiresAt <= now) { + return; + } + + const elapsedSeconds = Math.max(0, now - layer.lastUpdatedAt); + layer.lastUpdatedAt = now; + layer.styleIndex = styleIndex; + layer.energy = Math.max( + layer.energy * + Math.exp(-elapsedSeconds / generativePianoTuning.brushPhrase.energyDecaySeconds), + strength + ); + layer.maniaAmount = Math.max( + layer.maniaAmount * + Math.exp(-elapsedSeconds / generativePianoTuning.brushPhrase.maniaDecaySeconds), + maniaAmount + ); + layer.motifOffsets.push(this.getMotifOffset(strength)); + if (layer.motifOffsets.length > generativePianoTuning.brushMotifMaxSteps) { + layer.motifOffsets = layer.motifOffsets.slice( + -generativePianoTuning.brushMotifMaxSteps + ); + } + } + + private renderBrushPhraseLayers({ + vibe, + now, + lookaheadEnd, + activity, + }: { + vibe: VibePreset; + now: number; + lookaheadEnd: number; + activity: number; + }): void { + const earliestStart = now + PIANO_SCHEDULE_AHEAD_SECONDS; + this.nextBrushStreamStep ??= 0; + this.pruneBrushStreamNoteCounts(this.getGlobalBarIndex(now) - 1); + + this.brushPhraseLayers = this.brushPhraseLayers.filter( + (layer) => layer.expiresAt > earliestStart + ); + + while (this.getTimeForStep(this.nextBrushStreamStep) < earliestStart) { + const frame = this.getBrushStreamFrame( + this.getTimeForStep(this.nextBrushStreamStep), + activity + ); + this.nextBrushStreamStep += this.getBrushStreamIntervalSteps(frame.intensity); + this.brushStreamNoteIndex += 1; + } + + while (this.getTimeForStep(this.nextBrushStreamStep) <= lookaheadEnd) { + const startTime = this.getTimeForStep(this.nextBrushStreamStep); + const frame = this.getBrushStreamFrame(startTime, activity); + if ( + frame.intensity >= generativePianoTuning.brushLayerMinIntensity && + this.reserveBrushStreamNote(this.nextBrushStreamStep) + ) { + this.playBrushStreamNote({ + vibe, + startTime, + stepIndex: this.nextBrushStreamStep, + intensity: frame.intensity, + styleIndex: this.getStyleIndex(startTime), + layer: frame.layer, + }); + } + this.nextBrushStreamStep += this.getBrushStreamIntervalSteps(frame.intensity); + this.brushStreamNoteIndex += 1; + } + } + + private playBrushStreamNote({ + vibe, + startTime, + stepIndex, + intensity, + styleIndex, + layer, + }: { + vibe: VibePreset; + startTime: number; + stepIndex: number; + intensity: number; + styleIndex: GardenAudioStyleIndex; + layer: BrushPhraseLayer | null; + }): void { + const profile = getVibeProfile(vibe); + const pool = generativePianoTuning.stylePools[styleIndex]; + const maniaAmount = + layer?.maniaAmount ?? + clamp01( + (intensity - generativePianoTuning.brushStream.inferredManiaThreshold) / + generativePianoTuning.brushStream.inferredManiaRange + ); + const register = this.getBiasedRegister( + pool, + maniaAmount * generativePianoTuning.brushStream.registerManiaShift + ); + const chord = this.getChord(profile, this.getBarIndexForStep(stepIndex)); + const chordIntervals = getChordIntervals(chord, false); + const rootMidi = profile.rootMidi + chord.rootOffset; + const useChordTone = + this.brushStreamNoteIndex % + generativePianoTuning.brushStream.chordToneEverySteps === + 0; + const source = useChordTone + ? { + baseMidi: rootMidi, + offsets: this.getSupportOffsets(chordIntervals, styleIndex), + chordOffsets: chordIntervals, + } + : { + baseMidi: profile.rootMidi, + offsets: this.getBrushMotifDegrees({ + layer, + pool, + styleIndex, + }).map((degree) => degreeToSemitone(profile, degree)), + chordOffsets: this.getChordOffsets(chord, chordIntervals), + }; + const midi = this.chooseMidi(source, register, this.lastBrushStreamMidi, true); + const pan = this.getStylePan(styleIndex, intensity); + const durationSeconds = clamp( + generativePianoTuning.brushStream.durationBaseSeconds + + intensity * generativePianoTuning.brushStream.durationIntensitySeconds - + maniaAmount * generativePianoTuning.brushStream.durationManiaSeconds, + generativePianoTuning.brushStream.durationMinSeconds, + generativePianoTuning.brushStream.durationMaxSeconds + ); + const delaySend = clamp( + generativePianoTuning.brushStream.delaySendBase + + intensity * generativePianoTuning.brushStream.delaySendIntensityWeight - + maniaAmount * generativePianoTuning.brushStream.delaySendManiaWeight, + generativePianoTuning.brushStream.delaySendMin, + generativePianoTuning.brushStream.delaySendMax + ); + + this.lastBrushStreamMidi = midi; + this.lastMidiByStyle[styleIndex] = midi; + this.playProfileNote(profile, { + midi, + velocity: + (generativePianoTuning.brushStream.velocityBase + + intensity * generativePianoTuning.brushStream.velocityIntensityWeight) * + styleVoices[styleIndex].velocityMultiplier, + startTime, + durationSeconds, + pan, + role: 'brush', + delaySend, + lowpassHz: this.getLowpassHz( + profile, + midi, + clamp01( + generativePianoTuning.brushStream.lowpassBaseExpression + + intensity * generativePianoTuning.brushStream.lowpassIntensityWeight + + maniaAmount * generativePianoTuning.brushStream.lowpassManiaWeight + ) + ), + }); + + if ( + maniaAmount >= generativePianoTuning.brushStreamEcho.maniaThreshold && + (this.brushStreamNoteIndex % generativePianoTuning.brushStreamEcho.stepModulo === + generativePianoTuning.brushStreamEcho.stepRemainder || + intensity >= generativePianoTuning.brushStreamEcho.intensityThreshold) + ) { + const echoMidi = + midi + generativePianoTuning.brushStreamEcho.octaveSemitones <= + generativePianoTuning.brushStreamEcho.maxMidi + ? midi + generativePianoTuning.brushStreamEcho.octaveSemitones + : midi - generativePianoTuning.brushStreamEcho.octaveSemitones; + this.playProfileNote(profile, { + midi: echoMidi, + velocity: + (generativePianoTuning.brushStreamEcho.velocityBase + + intensity * generativePianoTuning.brushStreamEcho.velocityIntensityWeight) * + styleVoices[styleIndex].velocityMultiplier, + startTime: startTime + generativePianoTuning.brushMotifCanonDelaySeconds, + durationSeconds: Math.max( + generativePianoTuning.brushStreamEcho.durationMinSeconds, + durationSeconds * generativePianoTuning.brushStreamEcho.durationScale + ), + pan: clamp(pan * generativePianoTuning.brushStreamEcho.panScale, -1, 1), + role: 'brush', + delaySend: Math.max( + generativePianoTuning.brushStreamEcho.delaySendMin, + delaySend * generativePianoTuning.brushStreamEcho.delaySendScale + ), + lowpassHz: this.getLowpassHz( + profile, + echoMidi, + generativePianoTuning.brushStreamEcho.lowpassBaseExpression + + maniaAmount * generativePianoTuning.brushStreamEcho.lowpassManiaWeight + ), + }); + } + } + + private getBrushStreamFrame( + startTime: number, + activity: number + ): { + intensity: number; + layer: BrushPhraseLayer | null; + } { + const layerStates = this.brushPhraseLayers.map((layer) => ({ + layer, + intensity: + layer.energy * + this.getBrushPhraseFade(layer, startTime) * + (generativePianoTuning.brushPhrase.layerIntensityBase + + layer.maniaAmount * + generativePianoTuning.brushPhrase.layerIntensityManiaWeight), + })); + const dominant = layerStates.reduce<{ + layer: BrushPhraseLayer; + intensity: number; + } | null>((best, state) => { + if (state.intensity <= 0) { + return best; + } + return best === null || state.intensity > best.intensity ? state : best; + }, null); + const layeredIntensity = layerStates.reduce( + (sum, state) => sum + Math.max(0, state.intensity), + 0 + ); + + return { + intensity: clamp01( + activity * generativePianoTuning.brushPhrase.frameActivityWeight + + layeredIntensity + + (dominant?.layer.maniaAmount ?? 0) * + generativePianoTuning.brushPhrase.frameManiaWeight + ), + layer: dominant?.layer ?? null, + }; + } + + private getBrushStreamIntervalSteps(intensity: number): number { + const intervalBeats = + intensity >= generativePianoTuning.brushStream.intenseThreshold + ? generativePianoTuning.brushStreamIntenseIntervalBeats + : intensity >= generativePianoTuning.brushStream.activeThreshold + ? generativePianoTuning.brushStreamActiveIntervalBeats + : generativePianoTuning.brushStreamIdleIntervalBeats; + return Math.max(1, Math.round(intervalBeats * this.config.rhythm.stepsPerBeat)); + } + + private getBrushPhraseFade(layer: BrushPhraseLayer, startTime: number): number { + const lifetimeSeconds = Math.max(0.001, layer.expiresAt - layer.startedAt); + const ageSeconds = startTime - layer.startedAt; + return clamp01(1 - ageSeconds / lifetimeSeconds); + } + + private getMotifOffset(strength: number): number { + return strength >= generativePianoTuning.brushMotif.highThreshold + ? generativePianoTuning.brushMotif.highOffset + : strength >= generativePianoTuning.brushMotif.mediumThreshold + ? generativePianoTuning.brushMotif.mediumOffset + : generativePianoTuning.brushMotif.lowOffset; + } + + private getBrushMotifDegrees({ + layer, + pool, + styleIndex, + }: { + layer: BrushPhraseLayer | null; + pool: GardenAudioStylePool; + styleIndex: GardenAudioStyleIndex; + }): Array { + const styleOffset = styleVoices[styleIndex].scaleDegreeOffset; + if (!layer || layer.motifOffsets.length === 0) { + return this.rotate(pool.scaleDegrees, this.brushStreamNoteIndex + styleOffset); + } + + const motifOffset = + layer.motifOffsets[this.brushStreamNoteIndex % layer.motifOffsets.length]; + const baseOffset = styleOffset + motifOffset; + + return this.rotate( + pool.scaleDegrees.map((degree) => degree + baseOffset), + this.brushStreamNoteIndex + ); + } + + private getBiasedRegister( + register: GardenAudioRegister, + maniaAmount: number + ): GardenAudioRegister { + const shift = Math.round( + maniaAmount * generativePianoTuning.registerBias.maniaShiftSemitones + ); + const midiMin = clamp( + register.midiMin + shift, + generativePianoTuning.registerBias.midiMin, + generativePianoTuning.registerBias.midiMaxForMin + ); + const midiMax = clamp( + register.midiMax + shift, + midiMin + generativePianoTuning.registerBias.minimumSpan, + generativePianoTuning.registerBias.midiMax + ); + + return { + midiMin, + midiMax, + preferredMidi: clamp(register.preferredMidi + shift, midiMin, midiMax), + pan: register.pan, + }; + } + + private chooseMidi( + pitchSource: PitchSource, + register: GardenAudioRegister, + previousMidi: number | null = null, + avoidRepeat = false + ): number { + const candidates = this.getCandidates(pitchSource, register); + const referenceMidi = previousMidi ?? register.preferredMidi; + + if (candidates.length === 0) { + return register.preferredMidi; + } + + return candidates.reduce((best, candidate) => + this.scoreCandidate(candidate, register, referenceMidi, avoidRepeat) < + this.scoreCandidate(best, register, referenceMidi, avoidRepeat) + ? candidate + : best + ).midi; + } + + private getCandidates( + pitchSource: PitchSource, + register: GardenAudioRegister + ): Array { + const candidates: Array = []; + const chordPitchClasses = pitchSource.chordOffsets?.map((offset) => + getPitchClass(pitchSource.baseMidi + offset) + ); + + pitchSource.offsets.forEach((offset, preference) => { + for ( + let octave = generativePianoTuning.candidateOctaveSearch.min; + octave <= generativePianoTuning.candidateOctaveSearch.max; + octave += 1 + ) { + const midi = pitchSource.baseMidi + offset + octave * PITCH_SEMITONES_PER_OCTAVE; + if (midi >= register.midiMin && midi <= register.midiMax) { + const roundedMidi = Math.round(midi); + candidates.push({ + midi: roundedMidi, + preference, + chordToneDistance: getPitchClassDistance(roundedMidi, chordPitchClasses), + }); + } + } + }); + + return candidates; + } + + private scoreCandidate( + candidate: PitchCandidate, + register: GardenAudioRegister, + previousMidi: number, + avoidRepeat: boolean + ): number { + return ( + Math.abs(candidate.midi - previousMidi) + + Math.abs(candidate.midi - register.preferredMidi) * + generativePianoTuning.noteScoreRegisterWeight + + candidate.preference * generativePianoTuning.noteScorePreferenceWeight + + candidate.chordToneDistance * generativePianoTuning.noteScoreChordToneWeight + + (avoidRepeat && candidate.midi === previousMidi + ? generativePianoTuning.noteScoreRepeatPenalty + : 0) + ); + } + + private shouldPlaySupport(expression: number, barIndex: number): boolean { + if (expression < SUPPORT_ONSET_EXPRESSION) { + return false; + } + if (expression >= generativePianoTuning.supportNote.expressionThreshold) { + return true; + } + + return ( + barIndex % generativePianoTuning.supportBarSpacing === + generativePianoTuning.supportBarOffset + ); + } + + private shouldPlayTexture(expression: number, barIndex: number): boolean { + if (expression < TEXTURE_ONSET_EXPRESSION) { + return false; + } + if (expression >= generativePianoTuning.textureNote.mediumExpressionThreshold) { + return barIndex % generativePianoTuning.textureNote.intenseSpacing === 0; + } + const spacing = + expression < generativePianoTuning.textureNote.idleExpressionThreshold + ? generativePianoTuning.idleTextureBarSpacing + : generativePianoTuning.mediumTextureBarSpacing; + return barIndex % spacing === generativePianoTuning.textureNote.idlePhase % spacing; + } + + private getSupportOffsets( + chordIntervals: ReadonlyArray, + styleIndex: GardenAudioStyleIndex + ): Array { + return generativePianoTuning.supportNote.offsetsByStyle[styleIndex].map((offset) => + getConfiguredChordOffset(chordIntervals, offset) + ); + } + + private getChord(profile: GardenAudioVibeProfile, barIndex: number): GardenAudioChord { + const progressionIndex = + Math.floor(barIndex / generativePianoTuning.chordBars) % profile.progression.length; + return profile.progression[progressionIndex]; + } + + private getGlobalBarIndex(startTime: number): number { + return this.getBarIndexForStep(this.getStepIndexAtTime(startTime)); + } + + private getStyleIndex(startTime: number): GardenAudioStyleIndex { + const styleCount = generativePianoTuning.stylePools.length; + const rotationBars = Math.max(1, Math.round(generativePianoTuning.styleRotationBars)); + return (Math.floor(this.getGlobalBarIndex(startTime) / rotationBars) % + styleCount) as GardenAudioStyleIndex; + } + + private getStylePan(styleIndex: GardenAudioStyleIndex, activity: number): number { + const pool = generativePianoTuning.stylePools[styleIndex]; + const styleVoice = styleVoices[styleIndex]; + return this.getActivityPan( + pool.pan + styleVoice.panOffset * generativePianoTuning.stylePanOffsetScale, + activity + ); + } + + private getActivityPan(pan: number, activity: number): number { + const { active, idle, intense, intenseThreshold } = generativePianoTuning.stereoWidth; + const normalizedActivity = clamp01(activity); + const safeThreshold = clamp(intenseThreshold, 0.001, 0.999); + const width = + normalizedActivity < safeThreshold + ? idle + ((active - idle) * normalizedActivity) / safeThreshold + : active + + ((intense - active) * (normalizedActivity - safeThreshold)) / + (1 - safeThreshold); + + return clamp(pan * width, -1, 1); + } + + private getLowpassHz( + profile: GardenAudioVibeProfile, + midi: number, + expression: number + ): number { + const midiLift = + clamp01( + (midi - generativePianoTuning.lowpass.midiBase) / + generativePianoTuning.lowpass.midiRange + ) * generativePianoTuning.lowpass.midiLiftHz; + return clamp( + this.config.piano.lowpassHz * + profile.brightness * + (generativePianoTuning.lowpass.expressionBase + + expression * generativePianoTuning.lowpass.expressionWeight) + + midiLift, + this.config.piano.lowpassMinHz, + this.config.piano.lowpassMaxHz + ); + } + + private skipLateBeats(now: number): void { + if (this.nextBeatStep === null) { + return; + } + + const earliestStart = now + PIANO_SCHEDULE_AHEAD_SECONDS; + if (this.getTimeForStep(this.nextBeatStep) >= earliestStart) { + return; + } + + const earliestStep = this.getNextStepIndexAt(earliestStart, 0); + const stepsPerBeat = this.config.rhythm.stepsPerBeat; + this.nextBeatStep = Math.ceil(earliestStep / stepsPerBeat) * stepsPerBeat; + } + + private getExpression(activity: number): number { + const activityExpression = clamp01( + (activity - this.config.rhythm.sparseActivity) / + (1 - this.config.rhythm.sparseActivity) + ); + const idleExpression = clamp01( + this.activeProfile?.idleIntensity ?? this.config.rhythm.idleIntensity + ); + return Math.max(activityExpression, idleExpression); + } + + private getBeatDurationSeconds(): number { + return 60 / (this.activeProfile?.bpm ?? this.config.rhythm.bpm); + } + + private getStepDurationSeconds(): number { + return this.getBeatDurationSeconds() / this.config.rhythm.stepsPerBeat; + } + + private getBarDurationSeconds(): number { + return this.getBeatDurationSeconds() * this.getBeatsPerBar(); + } + + private getBeatsPerBar(): number { + return Math.max( + 1, + Math.round(this.config.rhythm.stepsPerBar / this.config.rhythm.stepsPerBeat) + ); + } + + private getTimeForStep(stepIndex: number): number { + return ( + (this.timelineStartedAt ?? 0) + + GENERATIVE_START_DELAY_SECONDS + + stepIndex * this.getStepDurationSeconds() + ); + } + + private getStepIndexAtTime(startTime: number): number { + const timelineStartedAt = this.timelineStartedAt ?? startTime; + const elapsedSeconds = Math.max( + 0, + startTime - timelineStartedAt - GENERATIVE_START_DELAY_SECONDS + ); + return Math.floor(elapsedSeconds / this.getStepDurationSeconds()); + } + + private getNextStepIndexAt(startTime: number, lookaheadSteps: number): number { + const timelineStartedAt = this.timelineStartedAt ?? startTime; + const elapsedSeconds = Math.max( + 0, + startTime - timelineStartedAt - GENERATIVE_START_DELAY_SECONDS + ); + return Math.max( + 0, + Math.ceil(elapsedSeconds / this.getStepDurationSeconds()) + lookaheadSteps + ); + } + + private getReleaseResolutionStep(startTime: number): number { + const currentStep = this.getStepIndexAtTime(startTime); + const nextBeatStep = + Math.ceil((currentStep + 1) / this.config.rhythm.stepsPerBeat) * + this.config.rhythm.stepsPerBeat; + const nextBarStep = + Math.ceil((currentStep + 1) / this.config.rhythm.stepsPerBar) * + this.config.rhythm.stepsPerBar; + const barIsClose = nextBarStep - currentStep <= this.config.rhythm.stepsPerBeat * 2; + + return barIsClose ? nextBarStep : nextBeatStep; + } + + private getNextBarTimeAt(startTime: number): number { + const nextBarStep = + Math.ceil(this.getStepIndexAtTime(startTime) / this.config.rhythm.stepsPerBar) * + this.config.rhythm.stepsPerBar; + return this.getTimeForStep(nextBarStep); + } + + private getBeatIndexForStep(stepIndex: number): number { + return Math.floor(stepIndex / this.config.rhythm.stepsPerBeat); + } + + private getBarIndexForStep(stepIndex: number): number { + return Math.floor(stepIndex / this.config.rhythm.stepsPerBar); + } + + private reserveBrushStreamNote(stepIndex: number): boolean { + const barIndex = this.getBarIndexForStep(stepIndex); + const noteCount = this.brushStreamNoteCountsByBar.get(barIndex) ?? 0; + if (noteCount >= generativePianoTuning.maxBrushStreamNotesPerBar) { + return false; + } + + this.brushStreamNoteCountsByBar.set(barIndex, noteCount + 1); + return true; + } + + private pruneBrushStreamNoteCounts(earliestBarIndex: number): void { + this.brushStreamNoteCountsByBar.forEach((_, barIndex) => { + if (barIndex < earliestBarIndex) { + this.brushStreamNoteCountsByBar.delete(barIndex); + } + }); + } + + private getChordOffsets( + chord: GardenAudioChord, + chordIntervals: ReadonlyArray + ): Array { + return chordIntervals.map((interval) => chord.rootOffset + interval); + } + + private rotate(values: ReadonlyArray, offset: number): Array { + return values.map((_, index) => values[(index + offset) % values.length]); + } +} + +const getPitchClass = (midi: number): number => ((midi % 12) + 12) % 12; + +const getPitchClassDistance = ( + midi: number, + chordPitchClasses: ReadonlyArray | undefined +): number => { + if (!chordPitchClasses || chordPitchClasses.length === 0) { + return 0; + } + + const pitchClass = getPitchClass(midi); + return chordPitchClasses.reduce((best, chordPitchClass) => { + const distance = Math.abs(pitchClass - chordPitchClass); + return Math.min(best, Math.min(distance, 12 - distance)); + }, 6); +}; + +const getConfiguredChordOffset = ( + chordIntervals: ReadonlyArray, + configuredOffset: number +): number => { + if (configuredOffset >= 12) { + const interval = chordIntervals[configuredOffset - 12] ?? 0; + return interval + 12; + } + + return chordIntervals[configuredOffset] ?? configuredOffset; +}; diff --git a/src/audio/noise-burst-player.ts b/src/audio/noise-burst-player.ts new file mode 100644 index 0000000..f9ab2bf --- /dev/null +++ b/src/audio/noise-burst-player.ts @@ -0,0 +1,64 @@ +import { clamp } from '../utils/math'; +import type { GardenAudioGraph } from './garden-audio-graph'; +import type { NoiseBurst } from './garden-audio-types'; + +const noiseBurstTuning = { + attackSeconds: 0.004, + filterQ: 1.4, + offsetRandomSeconds: 0.4, + scheduleAheadSeconds: 0.002, + silentGain: 0.0001, + filterType: 'bandpass', +} as const; + +export class NoiseBurstPlayer { + public constructor(private readonly graph: GardenAudioGraph) {} + + public play({ startTime, durationSeconds, gain, filterHz, pan }: NoiseBurst): void { + const { context, noiseBus, noiseBuffer } = this.graph; + if (!context || !noiseBus || !noiseBuffer) { + return; + } + + const scheduledStart = Math.max( + context.currentTime + noiseBurstTuning.scheduleAheadSeconds, + startTime + ); + const source = context.createBufferSource(); + const filter = context.createBiquadFilter(); + const envelope = context.createGain(); + const panner = context.createStereoPanner(); + const stopAt = scheduledStart + durationSeconds; + + source.buffer = noiseBuffer; + filter.type = noiseBurstTuning.filterType; + filter.frequency.setValueAtTime(filterHz, scheduledStart); + filter.Q.value = noiseBurstTuning.filterQ; + envelope.gain.setValueAtTime(noiseBurstTuning.silentGain, scheduledStart); + envelope.gain.exponentialRampToValueAtTime( + Math.max(noiseBurstTuning.silentGain, gain), + scheduledStart + noiseBurstTuning.attackSeconds + ); + envelope.gain.exponentialRampToValueAtTime(noiseBurstTuning.silentGain, stopAt); + panner.pan.setValueAtTime(clamp(pan, -1, 1), scheduledStart); + source.connect(filter); + filter.connect(envelope); + envelope.connect(panner); + panner.connect(noiseBus); + const maxOffsetSeconds = Math.max(0, noiseBuffer.duration - durationSeconds); + const offsetSeconds = + Math.random() * Math.min(noiseBurstTuning.offsetRandomSeconds, maxOffsetSeconds); + source.start(scheduledStart, offsetSeconds); + source.stop(stopAt); + source.addEventListener( + 'ended', + () => { + source.disconnect(); + filter.disconnect(); + envelope.disconnect(); + panner.disconnect(); + }, + { once: true } + ); + } +} diff --git a/src/audio/piano-sampler.ts b/src/audio/piano-sampler.ts new file mode 100644 index 0000000..1a0c77c --- /dev/null +++ b/src/audio/piano-sampler.ts @@ -0,0 +1,264 @@ +import { clamp, clamp01 } from '../utils/math'; +import type { GardenAudioConfig } from './garden-audio-config'; +import type { GardenAudioGraph } from './garden-audio-graph'; +import { PITCH_SEMITONES_PER_OCTAVE } from './garden-audio-music'; +import type { LoadedPianoSample, PianoNote } from './garden-audio-types'; +import { getLoadedPianoSamples, loadPianoSamples } from './piano-samples'; + +export const PIANO_SCHEDULE_AHEAD_SECONDS = 0.002; + +interface ActivePianoVoice { + gain: GainNode; + source: AudioScheduledSourceNode; + stopAt: number; +} + +const pianoSamplerTuning = { + filterType: 'lowpass', + filterQ: 0.7, + minDurationSeconds: 0.08, + minFadeSeconds: 0.08, + minGain: 0.0001, + releaseTimeConstantCount: 5, + tailStopExtraSeconds: 0.05, + voiceStealFadeSeconds: 0.025, + voiceStealStopSeconds: 0.05, +} as const; + +export class PianoSampler { + private samples: Array = []; + private activeVoices: Array = []; + + public constructor( + private readonly config: GardenAudioConfig, + private readonly graph: GardenAudioGraph + ) {} + + public load(context: BaseAudioContext): Promise { + if (this.samples.length > 0) { + return Promise.resolve(); + } + + const loadedSamples = getLoadedPianoSamples(); + if (loadedSamples) { + this.setSamples(loadedSamples); + return Promise.resolve(); + } + + return loadPianoSamples(context).then((samples) => { + this.setSamples(samples); + }); + } + + public play({ + midi, + velocity, + startTime, + durationSeconds, + pan, + role, + delaySend = 0, + lowpassHz = this.config.piano.lowpassHz, + sustainSeconds: profileSustainSeconds = this.config.piano.sustainSeconds, + }: PianoNote): void { + const { context } = this.graph; + const eventBus = this.graph.getPianoBus(role); + if (!context || !eventBus) { + return; + } + + const sample = this.findNearestSample(midi); + if (!sample) { + return; + } + + const scheduledStart = Math.max( + context.currentTime + PIANO_SCHEDULE_AHEAD_SECONDS, + startTime + ); + const noteVelocity = clamp01(velocity); + const noteGainValue = this.computeNoteGain(noteVelocity); + const sustainSeconds = + profileSustainSeconds * + (this.config.piano.sustainBase + + noteVelocity * this.config.piano.sustainVelocityRange); + const sustainAt = + scheduledStart + Math.max(pianoSamplerTuning.minDurationSeconds, durationSeconds); + const releaseAt = sustainAt + sustainSeconds; + const stopAt = + releaseAt + + this.config.piano.releaseSeconds * pianoSamplerTuning.releaseTimeConstantCount; + const source = context.createBufferSource(); + + source.buffer = sample.buffer; + source.playbackRate.setValueAtTime( + Math.pow(2, (midi - sample.midi) / PITCH_SEMITONES_PER_OCTAVE), + scheduledStart + ); + + this.scheduleVoice({ + source, + scheduledStart, + stopAt, + pan, + lowpassHz, + delaySend, + eventBus, + configureGainEnvelope: (gain) => { + gain.gain.setValueAtTime(pianoSamplerTuning.minGain, scheduledStart); + gain.gain.exponentialRampToValueAtTime( + noteGainValue, + scheduledStart + this.config.piano.gainAttackSeconds + ); + gain.gain.setTargetAtTime( + Math.max( + pianoSamplerTuning.minGain, + noteGainValue * this.config.piano.sustainLevel + ), + sustainAt, + Math.max( + pianoSamplerTuning.minFadeSeconds, + sustainSeconds * this.config.piano.sustainBase + ) + ); + gain.gain.setTargetAtTime( + pianoSamplerTuning.minGain, + releaseAt, + this.config.piano.releaseSeconds + ); + }, + }); + } + + public stopAll(): void { + const context = this.graph.context; + if (!context) { + this.activeVoices = []; + return; + } + + const now = context.currentTime; + + this.activeVoices.forEach((voice) => { + this.stopVoice(voice, now); + }); + this.activeVoices = []; + } + + public reset(): void { + this.samples = []; + this.activeVoices = []; + } + + private scheduleVoice({ + source, + scheduledStart, + stopAt, + pan, + lowpassHz, + delaySend, + eventBus, + configureGainEnvelope, + }: { + source: AudioScheduledSourceNode; + scheduledStart: number; + stopAt: number; + pan: number; + lowpassHz: number; + delaySend: number; + eventBus: GainNode; + configureGainEnvelope: (gain: GainNode) => void; + }): void { + const { context, delayInput } = this.graph; + if (!context) { + return; + } + + const filter = context.createBiquadFilter(); + const gain = context.createGain(); + const panner = context.createStereoPanner(); + let sendGain: GainNode | null = null; + + this.trimActiveVoices(scheduledStart); + while (this.activeVoices.length >= this.config.piano.maxVoices) { + const oldest = this.activeVoices.shift(); + if (!oldest) { + break; + } + this.stopVoice(oldest, scheduledStart); + } + + filter.type = pianoSamplerTuning.filterType; + filter.frequency.setValueAtTime( + clamp(lowpassHz, this.config.piano.lowpassMinHz, this.config.piano.lowpassMaxHz), + scheduledStart + ); + filter.Q.value = pianoSamplerTuning.filterQ; + panner.pan.setValueAtTime(clamp(pan, -1, 1), scheduledStart); + configureGainEnvelope(gain); + + source.connect(filter); + filter.connect(gain); + gain.connect(panner); + panner.connect(eventBus); + + if (delayInput && delaySend > 0) { + sendGain = context.createGain(); + sendGain.gain.value = delaySend; + panner.connect(sendGain); + sendGain.connect(delayInput); + } + + source.start(scheduledStart); + source.stop(stopAt + pianoSamplerTuning.tailStopExtraSeconds); + this.activeVoices.push({ gain, source, stopAt }); + + source.addEventListener( + 'ended', + () => { + source.disconnect(); + filter.disconnect(); + gain.disconnect(); + panner.disconnect(); + sendGain?.disconnect(); + this.activeVoices = this.activeVoices.filter((voice) => voice.gain !== gain); + }, + { once: true } + ); + } + + private computeNoteGain(velocity: number): number { + return Math.max(pianoSamplerTuning.minGain, this.config.piano.gain * velocity); + } + + private findNearestSample(midi: number): LoadedPianoSample | null { + if (this.samples.length === 0) { + return null; + } + + return this.samples.reduce((nearest, sample) => + Math.abs(sample.midi - midi) < Math.abs(nearest.midi - midi) ? sample : nearest + ); + } + + private trimActiveVoices(now: number): void { + this.activeVoices = this.activeVoices.filter((voice) => voice.stopAt > now); + } + + private stopVoice(voice: ActivePianoVoice, now: number): void { + const stopAt = now + pianoSamplerTuning.voiceStealStopSeconds; + + voice.gain.gain.cancelScheduledValues(now); + voice.gain.gain.setTargetAtTime( + pianoSamplerTuning.minGain, + now, + pianoSamplerTuning.voiceStealFadeSeconds + ); + voice.stopAt = stopAt; + voice.source.stop(stopAt); + } + + private setSamples(samples: Array): void { + this.samples = samples.slice().sort((a, b) => a.midi - b.midi); + } +} diff --git a/src/audio/piano-samples.ts b/src/audio/piano-samples.ts new file mode 100644 index 0000000..569eca4 --- /dev/null +++ b/src/audio/piano-samples.ts @@ -0,0 +1,271 @@ +import type { LoadedPianoSample } from './garden-audio-types'; +import a0SampleUrl from './samples/A0v12.m4a?url&no-inline'; +import a1SampleUrl from './samples/A1v12.m4a?url&no-inline'; +import a2SampleUrl from './samples/A2v12.m4a?url&no-inline'; +import a3SampleUrl from './samples/A3v12.m4a?url&no-inline'; +import a4SampleUrl from './samples/A4v12.m4a?url&no-inline'; +import a5SampleUrl from './samples/A5v12.m4a?url&no-inline'; +import a6SampleUrl from './samples/A6v12.m4a?url&no-inline'; +import a7SampleUrl from './samples/A7v12.m4a?url&no-inline'; +import c1SampleUrl from './samples/C1v12.m4a?url&no-inline'; +import c2SampleUrl from './samples/C2v12.m4a?url&no-inline'; +import c3SampleUrl from './samples/C3v12.m4a?url&no-inline'; +import c4SampleUrl from './samples/C4v12.m4a?url&no-inline'; +import c5SampleUrl from './samples/C5v12.m4a?url&no-inline'; +import c6SampleUrl from './samples/C6v12.m4a?url&no-inline'; +import c7SampleUrl from './samples/C7v12.m4a?url&no-inline'; +import c8SampleUrl from './samples/C8v12.m4a?url&no-inline'; +import dSharp1SampleUrl from './samples/Dsharp1v12.m4a?url&no-inline'; +import dSharp2SampleUrl from './samples/Dsharp2v12.m4a?url&no-inline'; +import dSharp3SampleUrl from './samples/Dsharp3v12.m4a?url&no-inline'; +import dSharp4SampleUrl from './samples/Dsharp4v12.m4a?url&no-inline'; +import dSharp5SampleUrl from './samples/Dsharp5v12.m4a?url&no-inline'; +import dSharp6SampleUrl from './samples/Dsharp6v12.m4a?url&no-inline'; +import dSharp7SampleUrl from './samples/Dsharp7v12.m4a?url&no-inline'; +import fSharp1SampleUrl from './samples/Fsharp1v12.m4a?url&no-inline'; +import fSharp2SampleUrl from './samples/Fsharp2v12.m4a?url&no-inline'; +import fSharp3SampleUrl from './samples/Fsharp3v12.m4a?url&no-inline'; +import fSharp4SampleUrl from './samples/Fsharp4v12.m4a?url&no-inline'; +import fSharp5SampleUrl from './samples/Fsharp5v12.m4a?url&no-inline'; +import fSharp6SampleUrl from './samples/Fsharp6v12.m4a?url&no-inline'; +import fSharp7SampleUrl from './samples/Fsharp7v12.m4a?url&no-inline'; + +interface PianoSampleDefinition { + note: string; + url: string; +} + +export interface PianoSampleLoadProgress { + failedCount: number; + loadedCount: number; + settledCount: number; + totalCount: number; +} + +const pianoSampleDefinitions: Array = [ + { url: a0SampleUrl, note: 'A0' }, + { url: c1SampleUrl, note: 'C1' }, + { url: dSharp1SampleUrl, note: 'Dsharp1' }, + { url: fSharp1SampleUrl, note: 'Fsharp1' }, + { url: a1SampleUrl, note: 'A1' }, + { url: c2SampleUrl, note: 'C2' }, + { url: dSharp2SampleUrl, note: 'Dsharp2' }, + { url: fSharp2SampleUrl, note: 'Fsharp2' }, + { url: a2SampleUrl, note: 'A2' }, + { url: c3SampleUrl, note: 'C3' }, + { url: dSharp3SampleUrl, note: 'Dsharp3' }, + { url: fSharp3SampleUrl, note: 'Fsharp3' }, + { url: a3SampleUrl, note: 'A3' }, + { url: c4SampleUrl, note: 'C4' }, + { url: dSharp4SampleUrl, note: 'Dsharp4' }, + { url: fSharp4SampleUrl, note: 'Fsharp4' }, + { url: a4SampleUrl, note: 'A4' }, + { url: c5SampleUrl, note: 'C5' }, + { url: dSharp5SampleUrl, note: 'Dsharp5' }, + { url: fSharp5SampleUrl, note: 'Fsharp5' }, + { url: a5SampleUrl, note: 'A5' }, + { url: c6SampleUrl, note: 'C6' }, + { url: dSharp6SampleUrl, note: 'Dsharp6' }, + { url: fSharp6SampleUrl, note: 'Fsharp6' }, + { url: a6SampleUrl, note: 'A6' }, + { url: c7SampleUrl, note: 'C7' }, + { url: dSharp7SampleUrl, note: 'Dsharp7' }, + { url: fSharp7SampleUrl, note: 'Fsharp7' }, + { url: a7SampleUrl, note: 'A7' }, + { url: c8SampleUrl, note: 'C8' }, +]; + +let loadedPianoSamples: Array | null = null; +let pianoSampleLoadPromise: Promise> | null = null; +let lastPianoSampleProgress: PianoSampleLoadProgress | null = null; +const pianoSampleProgressListeners = new Set< + (progress: PianoSampleLoadProgress) => void +>(); + +const sampleLoadTuning = { + concurrency: 4, + sampleTimeoutMs: 15_000, +}; + +export const preloadPianoSamples = ( + onProgress?: (progress: PianoSampleLoadProgress) => void +): Promise> => { + const OfflineAudioContextConstructor = globalThis.OfflineAudioContext; + + if (!OfflineAudioContextConstructor) { + return Promise.reject( + new Error('OfflineAudioContext is required to preload piano samples.') + ); + } + + // Decoding ignores these, but the constructor demands real numbers. + const decodeContext = new OfflineAudioContextConstructor(1, 1, 48_000); + return loadPianoSamples(decodeContext, onProgress); +}; + +export const loadPianoSamples = ( + decodeContext: BaseAudioContext, + onProgress?: (progress: PianoSampleLoadProgress) => void +): Promise> => { + const unsubscribeProgress = subscribeToPianoSampleProgress(onProgress); + + if (loadedPianoSamples) { + emitPianoSampleProgress({ + failedCount: 0, + loadedCount: loadedPianoSamples.length, + settledCount: loadedPianoSamples.length, + totalCount: pianoSampleDefinitions.length, + }); + unsubscribeProgress(); + return Promise.resolve([...loadedPianoSamples]); + } + + if (pianoSampleLoadPromise) { + return pianoSampleLoadPromise.finally(unsubscribeProgress); + } + + let loadedCount = 0; + let failedCount = 0; + let settledCount = 0; + const totalCount = pianoSampleDefinitions.length; + emitPianoSampleProgress({ failedCount, loadedCount, settledCount, totalCount }); + + pianoSampleLoadPromise = loadPianoSampleBatch( + pianoSampleDefinitions, + async (sample) => { + try { + const loadedSample = await withTimeout( + (signal) => loadPianoSample(decodeContext, sample, signal), + sampleLoadTuning.sampleTimeoutMs + ); + loadedCount += 1; + return loadedSample; + } catch (error) { + failedCount += 1; + throw error; + } finally { + settledCount += 1; + emitPianoSampleProgress({ failedCount, loadedCount, settledCount, totalCount }); + } + } + ) + .then( + (samples) => { + loadedPianoSamples = samples.sort((a, b) => a.midi - b.midi); + if (loadedPianoSamples.length !== pianoSampleDefinitions.length) { + throw new Error( + `Loaded ${loadedPianoSamples.length}/${pianoSampleDefinitions.length} piano samples.` + ); + } + return [...loadedPianoSamples]; + }, + (error: unknown) => { + pianoSampleLoadPromise = null; + pianoSampleProgressListeners.clear(); + throw error; + } + ) + .finally(unsubscribeProgress); + + return pianoSampleLoadPromise; +}; + +export const getLoadedPianoSamples = (): Array | null => + loadedPianoSamples ? [...loadedPianoSamples] : null; + +const loadPianoSample = async ( + decodeContext: BaseAudioContext, + sample: PianoSampleDefinition, + signal: AbortSignal +): Promise => { + const response = await fetch(sample.url, { signal }); + if (!response.ok) { + throw new Error(`Unable to load piano sample ${getPianoSamplePath(sample)}`); + } + + const audioData = await response.arrayBuffer(); + const buffer = await decodeContext.decodeAudioData(audioData); + return { midi: getMidiForPianoSample(sample), buffer }; +}; + +const loadPianoSampleBatch = async ( + samples: Array, + loadSample: (sample: PianoSampleDefinition) => Promise +): Promise> => { + const results: Array = []; + + for (let index = 0; index < samples.length; index += sampleLoadTuning.concurrency) { + const batch = samples.slice(index, index + sampleLoadTuning.concurrency); + const batchResults = await Promise.all(batch.map((sample) => loadSample(sample))); + results.push(...batchResults); + } + + return results; +}; + +const withTimeout = ( + operation: (signal: AbortSignal) => Promise, + timeoutMs: number +): Promise => + new Promise((resolve, reject) => { + const controller = new AbortController(); + const timeout = globalThis.setTimeout(() => { + controller.abort(); + reject(new Error('Timed out while loading a piano sample.')); + }, timeoutMs); + + operation(controller.signal).then( + (value) => { + globalThis.clearTimeout(timeout); + resolve(value); + }, + (error: unknown) => { + globalThis.clearTimeout(timeout); + reject(error); + } + ); + }); + +const subscribeToPianoSampleProgress = ( + onProgress: ((progress: PianoSampleLoadProgress) => void) | undefined +): (() => void) => { + if (!onProgress) { + return () => undefined; + } + + pianoSampleProgressListeners.add(onProgress); + if (lastPianoSampleProgress) { + onProgress(lastPianoSampleProgress); + } + return () => { + pianoSampleProgressListeners.delete(onProgress); + }; +}; + +const emitPianoSampleProgress = (progress: PianoSampleLoadProgress): void => { + lastPianoSampleProgress = progress; + pianoSampleProgressListeners.forEach((listener) => listener(progress)); +}; + +const getPianoSamplePath = (sample: PianoSampleDefinition): string => + `./samples/${sample.note}v12.m4a`; + +const getMidiForPianoSample = (sample: PianoSampleDefinition): number => { + const match = /^(?[A-G])(?sharp)?(?\d+)$/.exec(sample.note); + if (!match?.groups) { + throw new Error(`Invalid piano sample note ${sample.note}`); + } + + const semitoneByName: Record = { + C: 0, + D: 2, + E: 4, + F: 5, + G: 7, + A: 9, + B: 11, + }; + const octave = Number(match.groups.octave); + const semitone = semitoneByName[match.groups.name] + (match.groups.accidental ? 1 : 0); + return (octave + 1) * 12 + semitone; +}; diff --git a/src/audio/samples/A0v12.m4a b/src/audio/samples/A0v12.m4a new file mode 100644 index 0000000..db06fc3 Binary files /dev/null and b/src/audio/samples/A0v12.m4a differ diff --git a/src/audio/samples/A1v12.m4a b/src/audio/samples/A1v12.m4a new file mode 100644 index 0000000..f1ed488 Binary files /dev/null and b/src/audio/samples/A1v12.m4a differ diff --git a/src/audio/samples/A2v12.m4a b/src/audio/samples/A2v12.m4a new file mode 100644 index 0000000..52df725 Binary files /dev/null and b/src/audio/samples/A2v12.m4a differ diff --git a/src/audio/samples/A3v12.m4a b/src/audio/samples/A3v12.m4a new file mode 100644 index 0000000..707a766 Binary files /dev/null and b/src/audio/samples/A3v12.m4a differ diff --git a/src/audio/samples/A4v12.m4a b/src/audio/samples/A4v12.m4a new file mode 100644 index 0000000..679bcff Binary files /dev/null and b/src/audio/samples/A4v12.m4a differ diff --git a/src/audio/samples/A5v12.m4a b/src/audio/samples/A5v12.m4a new file mode 100644 index 0000000..4a2c896 Binary files /dev/null and b/src/audio/samples/A5v12.m4a differ diff --git a/src/audio/samples/A6v12.m4a b/src/audio/samples/A6v12.m4a new file mode 100644 index 0000000..abbd605 Binary files /dev/null and b/src/audio/samples/A6v12.m4a differ diff --git a/src/audio/samples/A7v12.m4a b/src/audio/samples/A7v12.m4a new file mode 100644 index 0000000..3fd6829 Binary files /dev/null and b/src/audio/samples/A7v12.m4a differ diff --git a/src/audio/samples/C1v12.m4a b/src/audio/samples/C1v12.m4a new file mode 100644 index 0000000..59d5f61 Binary files /dev/null and b/src/audio/samples/C1v12.m4a differ diff --git a/src/audio/samples/C2v12.m4a b/src/audio/samples/C2v12.m4a new file mode 100644 index 0000000..9b636f9 Binary files /dev/null and b/src/audio/samples/C2v12.m4a differ diff --git a/src/audio/samples/C3v12.m4a b/src/audio/samples/C3v12.m4a new file mode 100644 index 0000000..e891e16 Binary files /dev/null and b/src/audio/samples/C3v12.m4a differ diff --git a/src/audio/samples/C4v12.m4a b/src/audio/samples/C4v12.m4a new file mode 100644 index 0000000..6061dc5 Binary files /dev/null and b/src/audio/samples/C4v12.m4a differ diff --git a/src/audio/samples/C5v12.m4a b/src/audio/samples/C5v12.m4a new file mode 100644 index 0000000..a6d8898 Binary files /dev/null and b/src/audio/samples/C5v12.m4a differ diff --git a/src/audio/samples/C6v12.m4a b/src/audio/samples/C6v12.m4a new file mode 100644 index 0000000..745a4d6 Binary files /dev/null and b/src/audio/samples/C6v12.m4a differ diff --git a/src/audio/samples/C7v12.m4a b/src/audio/samples/C7v12.m4a new file mode 100644 index 0000000..6470854 Binary files /dev/null and b/src/audio/samples/C7v12.m4a differ diff --git a/src/audio/samples/C8v12.m4a b/src/audio/samples/C8v12.m4a new file mode 100644 index 0000000..dfbbfd1 Binary files /dev/null and b/src/audio/samples/C8v12.m4a differ diff --git a/src/audio/samples/Dsharp1v12.m4a b/src/audio/samples/Dsharp1v12.m4a new file mode 100644 index 0000000..22d0924 Binary files /dev/null and b/src/audio/samples/Dsharp1v12.m4a differ diff --git a/src/audio/samples/Dsharp2v12.m4a b/src/audio/samples/Dsharp2v12.m4a new file mode 100644 index 0000000..f25db22 Binary files /dev/null and b/src/audio/samples/Dsharp2v12.m4a differ diff --git a/src/audio/samples/Dsharp3v12.m4a b/src/audio/samples/Dsharp3v12.m4a new file mode 100644 index 0000000..7e09558 Binary files /dev/null and b/src/audio/samples/Dsharp3v12.m4a differ diff --git a/src/audio/samples/Dsharp4v12.m4a b/src/audio/samples/Dsharp4v12.m4a new file mode 100644 index 0000000..d670fbb Binary files /dev/null and b/src/audio/samples/Dsharp4v12.m4a differ diff --git a/src/audio/samples/Dsharp5v12.m4a b/src/audio/samples/Dsharp5v12.m4a new file mode 100644 index 0000000..cdbd7b8 Binary files /dev/null and b/src/audio/samples/Dsharp5v12.m4a differ diff --git a/src/audio/samples/Dsharp6v12.m4a b/src/audio/samples/Dsharp6v12.m4a new file mode 100644 index 0000000..b5ff787 Binary files /dev/null and b/src/audio/samples/Dsharp6v12.m4a differ diff --git a/src/audio/samples/Dsharp7v12.m4a b/src/audio/samples/Dsharp7v12.m4a new file mode 100644 index 0000000..a9b6cda Binary files /dev/null and b/src/audio/samples/Dsharp7v12.m4a differ diff --git a/src/audio/samples/Fsharp1v12.m4a b/src/audio/samples/Fsharp1v12.m4a new file mode 100644 index 0000000..752590f Binary files /dev/null and b/src/audio/samples/Fsharp1v12.m4a differ diff --git a/src/audio/samples/Fsharp2v12.m4a b/src/audio/samples/Fsharp2v12.m4a new file mode 100644 index 0000000..3477cb8 Binary files /dev/null and b/src/audio/samples/Fsharp2v12.m4a differ diff --git a/src/audio/samples/Fsharp3v12.m4a b/src/audio/samples/Fsharp3v12.m4a new file mode 100644 index 0000000..d36f8dd Binary files /dev/null and b/src/audio/samples/Fsharp3v12.m4a differ diff --git a/src/audio/samples/Fsharp4v12.m4a b/src/audio/samples/Fsharp4v12.m4a new file mode 100644 index 0000000..21df1e2 Binary files /dev/null and b/src/audio/samples/Fsharp4v12.m4a differ diff --git a/src/audio/samples/Fsharp5v12.m4a b/src/audio/samples/Fsharp5v12.m4a new file mode 100644 index 0000000..1105dfb Binary files /dev/null and b/src/audio/samples/Fsharp5v12.m4a differ diff --git a/src/audio/samples/Fsharp6v12.m4a b/src/audio/samples/Fsharp6v12.m4a new file mode 100644 index 0000000..d141d41 Binary files /dev/null and b/src/audio/samples/Fsharp6v12.m4a differ diff --git a/src/audio/samples/Fsharp7v12.m4a b/src/audio/samples/Fsharp7v12.m4a new file mode 100644 index 0000000..d69ac59 Binary files /dev/null and b/src/audio/samples/Fsharp7v12.m4a differ diff --git a/src/audio/samples/README.md b/src/audio/samples/README.md new file mode 100644 index 0000000..bdde746 --- /dev/null +++ b/src/audio/samples/README.md @@ -0,0 +1,16 @@ +Piano samples are Salamander Grand Piano V3 samples by Alexander Holm, +transcoded from OGG Vorbis to AAC M4A for iOS browser playback and distributed +under CC BY 3.0. + +Source package: @audio-samples/piano-velocity12 +Source recording: https://archive.org/details/SalamanderGrandPianoV3 +License: https://creativecommons.org/licenses/by/3.0/ + +Checked-in subset: velocity layer `v12`, every minor-third anchor from A0 +through C8: A, C, Dsharp, and Fsharp for octaves 1-7, plus A0, A7, and C8. +The app derives MIDI values from those note names in `piano-samples.ts`. + +Repro notes: start from the matching `v12` OGG files in the source package and +transcode each selected sample to AAC/M4A without renaming the note/velocity +stem. The expected output filenames are `v12.m4a`, for example +`C4v12.m4a`. diff --git a/src/page/audio-control.ts b/src/page/audio-control.ts new file mode 100644 index 0000000..7677bad --- /dev/null +++ b/src/page/audio-control.ts @@ -0,0 +1,152 @@ +import { appConfig } from '../config'; +import type GameLoop from '../game-loop/game-loop'; +import { readBrowserStorage, writeBrowserStorage } from '../utils/browser-storage'; +import { queryRequiredElement } from '../utils/dom'; +import { clamp01 } from '../utils/math'; + +const clampAudioVolume = (value: number): number => { + const { default: defaultVolume, max, min } = appConfig.toolbar.volume; + const safeValue = Number.isFinite(value) ? value : defaultVolume; + return Math.min(max, Math.max(min, clamp01(safeValue))); +}; + +const readInitialAudioVolume = (): number => { + const storedVolume = readBrowserStorage(appConfig.storage.audioVolumeKey); + return storedVolume === null + ? appConfig.toolbar.volume.default + : clampAudioVolume(Number(storedVolume)); +}; + +const formatStoredAudioVolume = (volume: number): string => + clampAudioVolume(volume).toFixed(2); + +const STORED_MUTED_TRUE = '1'; +const STORED_MUTED_FALSE = '0'; + +interface AudioControlOptions { + getGame: () => GameLoop | null; + hasStarted: () => boolean; + startButton: HTMLElement; +} + +export class AudioControl { + private readonly soundButton = queryRequiredElement( + '[data-control="sound"]', + HTMLButtonElement + ); + private readonly volumeControl = queryRequiredElement( + '.volume-control', + HTMLLabelElement + ); + private readonly volumeSlider = queryRequiredElement( + '.volume-slider', + HTMLInputElement + ); + + private audioVolume = readInitialAudioVolume(); + private isMutedState = + readBrowserStorage(appConfig.storage.audioMutedKey) === STORED_MUTED_TRUE || + this.audioVolume <= 0; + + public constructor(private readonly options: AudioControlOptions) { + this.soundButton.addEventListener('click', this.onToggleMute); + this.volumeSlider.addEventListener('input', this.onVolumeInput); + + const passiveCaptureOptions = { capture: true, passive: true } as const; + const captureOptions = { capture: true } as const; + ( + [ + ['touchstart', passiveCaptureOptions], + ['pointerdown', passiveCaptureOptions], + ['touchend', passiveCaptureOptions], + ['pointerup', passiveCaptureOptions], + ['click', captureOptions], + ['keydown', captureOptions], + ] satisfies Array<[keyof WindowEventMap, AddEventListenerOptions]> + ).forEach(([event, opts]) => { + window.addEventListener(event, this.onUserGesture, opts); + }); + + this.render(); + } + + public get isMuted(): boolean { + return this.isMutedState || this.audioVolume <= 0; + } + + public render(): void { + this.audioVolume = clampAudioVolume(this.audioVolume); + const isEffectivelyMuted = this.isMuted; + const volumePercent = Math.round(this.audioVolume * 100); + + this.soundButton.classList.toggle('muted', isEffectivelyMuted); + this.soundButton.setAttribute('aria-pressed', String(isEffectivelyMuted)); + const muteLabel = isEffectivelyMuted ? 'Unmute audio' : 'Mute audio'; + this.soundButton.setAttribute('aria-label', muteLabel); + this.soundButton.title = muteLabel; + + this.volumeSlider.min = appConfig.toolbar.volume.min.toString(); + this.volumeSlider.max = appConfig.toolbar.volume.max.toString(); + this.volumeSlider.step = appConfig.toolbar.volume.step.toString(); + this.volumeSlider.value = formatStoredAudioVolume(this.audioVolume); + this.volumeSlider.setAttribute( + 'aria-valuetext', + isEffectivelyMuted ? `Muted, ${volumePercent}%` : `${volumePercent}%` + ); + this.volumeControl.classList.toggle('muted', isEffectivelyMuted); + this.volumeControl.title = isEffectivelyMuted + ? `Muted, ${volumePercent}% volume` + : `${volumePercent}% volume`; + this.volumeControl.style.setProperty('--volume-progress', `${volumePercent}%`); + + const game = this.options.getGame(); + game?.setAudioVolume(this.audioVolume); + game?.setAudioMuted(isEffectivelyMuted); + } + + private readonly onToggleMute = () => { + const shouldUnmute = this.isMutedState || this.audioVolume <= 0; + if (shouldUnmute && this.audioVolume <= 0) { + this.audioVolume = appConfig.toolbar.volume.default; + } + this.isMutedState = !shouldUnmute; + this.persist(); + this.render(); + if (!this.isMutedState) { + this.options.getGame()?.startAudio(true); + } + }; + + private readonly onVolumeInput = () => { + this.audioVolume = clampAudioVolume(Number(this.volumeSlider.value)); + this.isMutedState = this.audioVolume <= 0; + this.persist(); + this.render(); + if (!this.isMutedState) { + this.options.getGame()?.startAudio(true); + } + }; + + private readonly onUserGesture = (event: Event) => { + if ( + !this.options.hasStarted() || + this.isMutedState || + (event.target instanceof Node && this.options.startButton.contains(event.target)) || + (event.target instanceof Node && this.soundButton.contains(event.target)) + ) { + return; + } + this.options.getGame()?.startAudio(true); + }; + + private persist(): void { + writeBrowserStorage( + appConfig.storage.audioMutedKey, + this.isMutedState ? STORED_MUTED_TRUE : STORED_MUTED_FALSE + ); + writeBrowserStorage( + appConfig.storage.audioVolumeKey, + formatStoredAudioVolume(this.audioVolume) + ); + } +}