From 968ab20b4001f245845c9eca4294e8c2bf4c2145 Mon Sep 17 00:00:00 2001 From: TP Date: Wed, 29 Oct 2025 01:10:41 -0700 Subject: [PATCH 1/2] Refactor TTS processing to use interval loop --- index.html | 74 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/index.html b/index.html index dd26df3..eee1620 100644 --- a/index.html +++ b/index.html @@ -1163,9 +1163,13 @@

🌐 OpenRouter Settings

// Update VRM expression based on UI controls updateVRMExpression(); - // Process TTS audio in main loop instead of separate RAF + // Ensure TTS processing loop matches playback state without heavy RAF work if (isTTSPlaying) { - processTTSAudio(); + if (!ttsProcessingIntervalId) { + startTTSAudioProcessingLoop(); + } + } else if (ttsProcessingIntervalId) { + stopTTSAudioProcessingLoop(); } // PERFORMANCE: Update expressions ONCE per frame after all setValue calls @@ -1870,7 +1874,7 @@

🌐 OpenRouter Settings

`; console.log('🎭 Starting Azure SDK TTS with visemes'); - isTTSPlaying = true; + setTTSPlayingState(true); isProcessingTTS = true; isUsingVisemes = true; @@ -1931,7 +1935,7 @@

🌐 OpenRouter Settings

audioElement.onended = () => { URL.revokeObjectURL(audioUrl); - isTTSPlaying = false; + setTTSPlayingState(false); isProcessingTTS = false; isUsingVisemes = false; @@ -1947,7 +1951,7 @@

🌐 OpenRouter Settings

} else { console.error('❌ Speech SDK: Error synthesizing speech:', result.errorDetails); // Only stop on error - isTTSPlaying = false; + setTTSPlayingState(false); isProcessingTTS = false; isUsingVisemes = false; stopTalkingAnimationForTTS(); @@ -1956,7 +1960,7 @@

🌐 OpenRouter Settings

}, error => { console.error('❌ Speech SDK: SpeakSsmlAsync error:', error); - isTTSPlaying = false; + setTTSPlayingState(false); isProcessingTTS = false; isUsingVisemes = false; stopTalkingAnimationForTTS(); @@ -1992,7 +1996,7 @@

🌐 OpenRouter Settings

if (currentTTSAudio) { currentTTSAudio.pause(); currentTTSAudio.currentTime = 0; - isTTSPlaying = false; + setTTSPlayingState(false); ttsInputVolume = 0; } @@ -2001,7 +2005,7 @@

🌐 OpenRouter Settings

currentTTSAudio.src = URL.createObjectURL(ttsResult.audioBlob); currentTTSAudio.crossOrigin = 'anonymous'; - isTTSPlaying = true; + setTTSPlayingState(true); isProcessingTTS = true; isUsingVisemes = true; startTalkingAnimationForTTS(); @@ -2013,7 +2017,7 @@

🌐 OpenRouter Settings

currentTTSAudio.onended = () => { URL.revokeObjectURL(currentTTSAudio.src); - isTTSPlaying = false; + setTTSPlayingState(false); isProcessingTTS = false; isUsingVisemes = false; stopTalkingAnimationForTTS(); @@ -2027,7 +2031,7 @@

🌐 OpenRouter Settings

currentTTSAudio.onerror = () => { console.error('Azure TTS audio playback failed'); - isTTSPlaying = false; + setTTSPlayingState(false); isProcessingTTS = false; isUsingVisemes = false; stopTalkingAnimationForTTS(); @@ -2503,19 +2507,19 @@

🌐 OpenRouter Settings

u.rate = azureConfig.rate||1.0; u.pitch = Math.max(0, Math.min(2, ((azureConfig.pitch||0)+12)/24)); u.volume = azureConfig.volume||0.9; u.onstart = ()=>{ console.log('🎤 Browser TTS started'); - isTTSPlaying = true; + setTTSPlayingState(true); isBrowserTTSActive = true; startTalkingAnimationForTTS(); // TTS audio processing now handled by main animation loop }; u.onend = ()=>{ - isTTSPlaying = false; + setTTSPlayingState(false); isBrowserTTSActive = false; stopTalkingAnimationForTTS(); resolve(); }; u.onerror = ()=>{ - isTTSPlaying = false; + setTTSPlayingState(false); isBrowserTTSActive = false; stopTalkingAnimationForTTS(); resolve(); @@ -2542,6 +2546,39 @@

🌐 OpenRouter Settings

// TTS audio processing now handled by main animation loop } + + // Dedicated interval to process TTS audio outside the render loop + function startTTSAudioProcessingLoop() { + if (ttsProcessingIntervalId) return; + + ttsProcessingIntervalId = setInterval(() => { + if (!isTTSPlaying) { + stopTTSAudioProcessingLoop(); + return; + } + processTTSAudio(); + }, TTS_PROCESS_INTERVAL_MS); + } + + function stopTTSAudioProcessingLoop() { + if (ttsProcessingIntervalId) { + clearInterval(ttsProcessingIntervalId); + ttsProcessingIntervalId = null; + } + } + + function setTTSPlayingState(playing) { + if (isTTSPlaying === playing) return; + + isTTSPlaying = playing; + + if (playing) { + startTTSAudioProcessingLoop(); + } else { + stopTTSAudioProcessingLoop(); + } + } + function processTTSAudio() { if (isTTSPlaying) { let ttsInputVolume = 0; @@ -3253,6 +3290,8 @@

🌐 OpenRouter Settings

let ttsInputVolume = 0; let currentTTSAudio = null; let isTTSPlaying = false; + const TTS_PROCESS_INTERVAL_MS = Math.round(1000 / 30); // ~30 Hz update cadence for TTS processing + let ttsProcessingIntervalId = null; let isProcessingTTS = false; let isBrowserTTSActive = false; let isUsingVisemes = false; // Flag to disable audio-reactive movement when using precise visemes @@ -3964,7 +4003,7 @@

🌐 OpenRouter Settings

if (isPlayingAudio || audioQueue.length === 0) return; isPlayingAudio = true; - isTTSPlaying = true; + setTTSPlayingState(true); // Start talking animation once at the beginning startTalkingAnimationForTTS(); @@ -3985,7 +4024,7 @@

🌐 OpenRouter Settings

// Queue is now empty - stop everything isPlayingAudio = false; isProcessingTTS = false; - isTTSPlaying = false; + setTTSPlayingState(false); isUsingVisemes = false; stopTalkingAnimationForTTS(); applyVisemeToVRM(vrm || currentVrm, 0, 0); @@ -5690,7 +5729,7 @@

🌐 OpenRouter Settings

if (currentTTSAudio) { currentTTSAudio.pause(); currentTTSAudio.currentTime = 0; - isTTSPlaying = false; + setTTSPlayingState(false); ttsInputVolume = 0; } @@ -5704,11 +5743,12 @@

🌐 OpenRouter Settings

// Setup audio processing for VRM mouth movement setupTTSAudioProcessing(currentTTSAudio); + setTTSPlayingState(true); await currentTTSAudio.play(); currentTTSAudio.onended = () => { URL.revokeObjectURL(audioUrl); - isTTSPlaying = false; + setTTSPlayingState(false); ttsInputVolume = 0; isProcessingTTS = false; updateButtonStates(); From 9efe4f2eb62c316487a46766e1b833f613268ec9 Mon Sep 17 00:00:00 2001 From: TP Date: Wed, 29 Oct 2025 01:25:48 -0700 Subject: [PATCH 2/2] Add Edge TTS provider option with streaming playback --- index.html | 315 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 291 insertions(+), 24 deletions(-) diff --git a/index.html b/index.html index eee1620..30dfff4 100644 --- a/index.html +++ b/index.html @@ -232,6 +232,15 @@

Settings

Enable Text-to-Speech +
+ + +
@@ -248,6 +257,7 @@

Settings