fix: full review pass (main process, voice pipeline, Hermes client, HUD) and e2e-validated local voice (v2.1.0.1)

Electron main
- utility-process channel: sherpa output copied into V8 buffers (Electron rejects external
  buffers), audio exchanged as base64; worker restart on timeout, identity-safe exit handling,
  deterministic native unload by restart
- webhook: loopback by default without secret, UTF-8-safe body assembly, clean restart, port validation
- files IPC: realpath-based root check, openPath allow-list (reveal-only outside document folders),
  Windows reserved names; store: debounced async atomic writes with backup of corrupt files;
  logger: streaming writes with rotation; log message size cap
- HTTP stream proxy: socket released on idle timeout / renderer destroyed, id validation
- model manager: inactivity timeout, retrying rm/rename (Windows locks), engine stopped before
  replacing a model; WAV decoder handles float/24-bit; IPC payload validation
- window: opaque rounded window on Windows, navigation lock-down, visibility events;
  Ctrl+Alt+Escape instead of the Task Manager shortcut; single-instance guard; EVEFLOW_USER_DATA

Voice pipeline
- abort semantics (SendHandle.aborted), abort before the stream opens, session id prefixes per transport
- hands-free re-arm after replies without speech, start/stop race, no chime on auto re-arm,
  no silence shipped to STT (400 ms pre-roll), no transcription of empty manual stops
- TTS: bounded prefetch, cancellable segments, non-interrupting notices, volume applied at play time
- SSE CRLF split, usage in chat completions, finish_reason length, phonetic regex hoisted

HUD
- core renderer: no canvas shadows, cached colours, reusable spectrum buffer, theme read on change,
  30 fps idle, stops when the window is hidden; ping flashes on send / tool / speech
- deltas coalesced per animation frame; stable auto-scroll; narrow selectors everywhere
- bundled fonts (offline), reduce-motion fix, error toasts, ops drawer below 1180 px,
  interim transcript and first-token latency in the core caption, Ctrl+K, dialog semantics,
  switch/aria roles, compact widget cleanup, Whisper small recommended for French
- docs/ROADMAP.md: audit results, e2e results and the plan towards a real JARVIS

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y
This commit is contained in:
Claude committed 2026-09-03 18:08:09 +00:00
1 parent b1b8d0d5d4
commit 5f0fa7a1e2
55 files changed
+1181 -433

No files matched your search

+62 -36
View File
@@ -1,7 +1,7 @@
/**
* Text-to-speech engine with a sentence queue, prefetching and Web Audio playback so the HUD
* reacts to the actual waveform. Providers: OpenAI-compatible /v1/audio/speech, system voices,
* and the legacy Google Translate endpoint (no key, online only).
* Text-to-speech engine with a sentence queue, bounded prefetching and Web Audio playback so
* the HUD reacts to the actual waveform. Providers: in-app sherpa-onnx (Kokoro / Piper),
* OpenAI-compatible /v1/audio/speech, system voices, and the legacy Google Translate endpoint.
*/
import { Log } from '../../lib/log';
import { bridge } from '../../lib/bridge';
@@ -35,6 +35,8 @@ interface QueueItem {
audio?: Promise<AudioBuffer | null>;
}
const LOOKAHEAD = 2;
export function ttsEndpoint(apiUrl: string): string {
const base = apiUrl.trim().replace(/\/+$/, '');
if (base.endsWith('/speech')) return base;
@@ -46,8 +48,7 @@ export function ttsEndpoint(apiUrl: string): string {
export class TtsEngine {
private queue: QueueItem[] = [];
private playing = false;
private currentSource: AudioBufferSourceNode | null = null;
private currentUtterance: SpeechSynthesisUtterance | null = null;
private cancelCurrent: (() => void) | null = null;
private streamBuffer = '';
private generation = 0;
private config: TtsConfig;
@@ -60,7 +61,6 @@ export class TtsEngine {
updateConfig(config: TtsConfig): void {
this.config = config;
audioBus.setVolume(config.volume);
}
onState(listener: (state: TtsState) => void): () => void {
@@ -78,10 +78,18 @@ export class TtsEngine {
for (const l of this.listeners) l(state);
}
/** Speak a finished text (clears any pending speech first). */
speak(text: string): void {
this.stop();
for (const chunk of chunkForSpeech(text)) this.enqueue(chunk);
/**
* Speak a finished text. With `interrupt` (default) pending speech is discarded first;
* without it the text is inserted ahead of the queue and streamed speech resumes afterwards.
*/
speak(text: string, options: { interrupt?: boolean } = {}): void {
const interrupt = options.interrupt ?? true;
if (interrupt) this.stop();
const items = chunkForSpeech(text).map((chunk) => this.makeItem(chunk)).filter((i): i is QueueItem => !!i);
if (interrupt) this.queue.push(...items);
else this.queue.unshift(...items);
this.fillPrefetch();
void this.drain();
}
/** Feed streamed tokens; complete sentences are spoken as soon as they are available. */
@@ -100,32 +108,32 @@ export class TtsEngine {
}
enqueue(text: string): void {
if (this.config.provider === 'off') return;
const clean = cleanForSpeech(text);
if (!clean || !/[\p{L}\p{N}]/u.test(clean)) return;
const item: QueueItem = { text: clean };
if (this.config.provider !== 'system') item.audio = this.prefetch(clean, this.generation);
const item = this.makeItem(text);
if (!item) return;
this.queue.push(item);
this.fillPrefetch();
void this.drain();
}
private makeItem(text: string): QueueItem | null {
if (this.config.provider === 'off') return null;
const clean = cleanForSpeech(text);
if (!clean || !/[\p{L}\p{N}]/u.test(clean)) return null;
return { text: clean };
}
/** Keep at most LOOKAHEAD synthesis requests in flight ahead of playback. */
private fillPrefetch(): void {
if (this.config.provider === 'system') return;
for (const item of this.queue.slice(0, LOOKAHEAD)) item.audio ??= this.prefetch(item.text, this.generation);
}
stop(): void {
this.generation++;
this.queue = [];
this.streamBuffer = '';
if (this.currentSource) {
try {
this.currentSource.onended = null;
this.currentSource.stop();
} catch {
/* already stopped */
}
this.currentSource = null;
}
if (this.currentUtterance) {
this.currentUtterance.onend = null;
this.currentUtterance = null;
}
this.cancelCurrent?.();
this.cancelCurrent = null;
if (typeof speechSynthesis !== 'undefined') speechSynthesis.cancel();
audioBus.setSyntheticLevel(0);
this.playing = false;
@@ -138,15 +146,17 @@ export class TtsEngine {
const gen = this.generation;
while (this.queue.length > 0 && gen === this.generation) {
const item = this.queue.shift()!;
this.fillPrefetch();
try {
if (item.audio) {
if (this.config.provider !== 'system') {
this.setState('loading');
const buffer = await item.audio;
const buffer = await (item.audio ?? this.prefetch(item.text, gen));
if (gen !== this.generation) break;
if (buffer) {
this.setState('speaking');
await this.playBuffer(buffer, gen);
} else {
this.setState('speaking');
await this.speakSystem(item.text, gen);
}
} else {
@@ -174,7 +184,7 @@ export class TtsEngine {
return audioBus.context.decodeAudioData(copy);
})
.catch((err) => {
Log.warn('tts', `synthesis failed, falling back to system voice: ${(err as Error).message}`);
if (gen === this.generation) Log.warn('tts', `synthesis failed, falling back to system voice: ${(err as Error).message}`);
return null;
});
}
@@ -238,15 +248,26 @@ export class TtsEngine {
return new Promise((resolve) => {
if (gen !== this.generation) return resolve();
const ctx = audioBus.context;
audioBus.setVolume(this.config.volume);
const source = ctx.createBufferSource();
source.buffer = buffer;
if (this.config.provider === 'google-free') source.playbackRate.value = Math.max(0.5, Math.min(2, this.config.speed || 1));
source.connect(audioBus.output);
source.onended = () => {
if (this.currentSource === source) this.currentSource = null;
const finish = () => {
if (this.cancelCurrent === cancel) this.cancelCurrent = null;
resolve();
};
this.currentSource = source;
const cancel = () => {
source.onended = null;
try {
source.stop();
} catch {
/* already stopped */
}
finish();
};
source.onended = finish;
this.cancelCurrent = cancel;
source.start();
});
}
@@ -268,12 +289,17 @@ export class TtsEngine {
if (pulse) clearInterval(pulse);
pulse = null;
audioBus.setSyntheticLevel(0);
if (this.currentUtterance === utterance) this.currentUtterance = null;
if (this.cancelCurrent === cancel) this.cancelCurrent = null;
resolve();
};
const cancel = () => {
utterance.onend = null;
utterance.onerror = null;
finish();
};
utterance.onend = finish;
utterance.onerror = finish;
this.currentUtterance = utterance;
this.cancelCurrent = cancel;
speechSynthesis.speak(utterance);
});
}