feat: always-on wake word with sherpa-onnx keyword spotting (v2.2.0)

- Catalog: 3.3 MB zipformer keyword-spotting model (kws-en).
- shared/keywords: BPE encoding of wake phrases (SentencePiece table for common words,
  greedy longest-match fallback over the model vocabulary) and keywords file builder.
- Worker: KeywordSpotter stream fed with 16-bit PCM, detections pushed as unsolicited
  messages; engine derives the keywords file, maps sensitivity to threshold/score,
  forwards detections to the renderer and re-arms after a worker restart.
- Renderer: WakeListener keeps one microphone stream, batches 256 ms frames to the
  spotter and captures the command on the same stream after detection (pre-roll, VAD),
  then resumes spotting; the wake word also interrupts speech. Settings: wake mode
  (off / always-on / transcript filter), keyword, sensitivity, status and one-click
  model download; HUD caption shows the active keyword.
- Validated: detection in the worker (fork) and through the real Electron IPC path on
  Kokoro audio, no false positive on an English recording; 23 unit tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y

src/services/voice/voiceController.ts

@@ -10,7 +10,9 @@ import { audioBus } from './audioBus';
 import { listMicrophones, MicCapture } from './capture';
 import { speech } from './speech';
 import { BrowserRecognizer, transcribeWav } from './stt';
+import { WakeListener } from './wakeListener';
 import type { WavResult } from './wav';
+import { bridge } from '../../lib/bridge';
 
 const SENSITIVITY_RATIO: Record<number, number> = { 1: 4.5, 2: 3.4, 3: 2.6, 4: 2.0, 5: 1.6 };
 const SENSITIVITY_MIN_RMS: Record<number, number> = { 1: 0.03, 2: 0.02, 3: 0.012, 4: 0.008, 5: 0.005 };
@@ -46,6 +48,8 @@ export function matchWakeWord(text: string, wakeWord: string): { matched: boolea
 
 class VoiceController {
   private capture = new MicCapture();
+  private wake = new WakeListener();
+  private unsubscribeKws: (() => void) | null = null;
   /** After a bare wake word, the next utterance is accepted without the wake word. */
   private attentionUntil = 0;
   private browser = new BrowserRecognizer();
@@ -74,17 +78,104 @@ class VoiceController {
   dispose(): void {
     this.unsubscribeVoice?.();
     this.stop();
+    void this.stopWakeMode();
   }
 
   get isListening(): boolean {
     return useVoice.getState().phase !== 'off';
   }
 
+  get wakeActive(): boolean {
+    return this.wake.isActive;
+  }
+
   toggle(): void {
+    if (this.wake.isActive) {
+      // Always-on mode: the button starts/cancels a command capture on the shared microphone.
+      if (this.wake.currentPhase === 'command' || this.wake.currentPhase === 'speech') this.wake.cancelCommand();
+      else this.onWakeDetected('manuel');
+      return;
+    }
     if (this.isListening) this.stop();
     else void this.start();
   }
 
+  /** Always-on keyword spotting: the microphone stays open and the main process spots the wake word. */
+  async startWakeMode(): Promise<void> {
+    const api = bridge();
+    const voice = useVoice.getState();
+    if (!api || this.wake.isActive) return;
+    const settings = useSettings.getState().settings.voice;
+    voice.setWake('starting');
+    try {
+      const phrases = [settings.wakeWord || 'jarvis', `hey ${settings.wakeWord || 'jarvis'}`];
+      const result = await api.voice.kwsStart({ modelId: 'kws-en', keywords: phrases, sensitivity: settings.kwsSensitivity });
+      this.unsubscribeKws?.();
+      this.unsubscribeKws = api.voice.onKwsDetected((d) => this.onWakeDetected(d.keyword));
+      const sensitivity = Math.min(5, Math.max(1, Math.round(settings.sensitivity))) as 1 | 2 | 3 | 4 | 5;
+      await this.wake.start({
+        deviceId: settings.micDeviceId || undefined,
+        vad: { silenceMs: settings.silenceMs, speechRatio: SENSITIVITY_RATIO[sensitivity], minRms: SENSITIVITY_MIN_RMS[sensitivity] },
+        callbacks: {
+          onPhase: (phase) => {
+            const v = useVoice.getState();
+            if (phase === 'spotting') {
+              v.setPhase('off');
+              const chat = useChat.getState();
+              if (chat.hud === 'listening') chat.setHud(chat.isSending ? 'thinking' : 'idle');
+            } else if (phase === 'command') {
+              v.setPhase('listening');
+              useChat.getState().setHud('listening');
+            } else if (phase === 'speech') {
+              v.setPhase('speech');
+              useChat.getState().ping();
+            }
+          },
+          onLevel: (level) => {
+            const q = Math.round(level * 20) / 20;
+            if (q !== useVoice.getState().inputLevel) useVoice.getState().setInputLevel(q);
+          },
+          onUtterance: (wav) => void this.transcribe(wav, true),
+          onNoSpeech: () => useVoice.getState().setInputLevel(0),
+          onError: (message) => useVoice.getState().setError(message)
+        }
+      });
+      voice.setWake('spotting', result.accepted.map((k) => k.replace(/_/g, ' ')));
+      Log.info('voice', `wake mode on: ${result.accepted.join(', ')}${result.rejected.length ? ` (rejetés : ${result.rejected.join(', ')})` : ''}`);
+    } catch (err) {
+      const message = (err as Error).message;
+      voice.setWake('error');
+      voice.setError(message);
+      useChat.getState().setError(`Mot d’activation : ${message}`);
+      Log.error('voice', `wake mode failed: ${message}`);
+      await api.voice.kwsStop().catch(() => undefined);
+    }
+  }
+
+  async stopWakeMode(): Promise<void> {
+    this.unsubscribeKws?.();
+    this.unsubscribeKws = null;
+    this.wake.stop();
+    useVoice.getState().setWake('off');
+    await bridge()?.voice.kwsStop().catch(() => undefined);
+  }
+
+  /** Restart spotting with the current settings (wake word, sensitivity, microphone). */
+  async restartWakeMode(): Promise<void> {
+    await this.stopWakeMode();
+    if (useSettings.getState().settings.voice.wakeMode === 'kws') await this.startWakeMode();
+  }
+
+  private onWakeDetected(keyword: string): void {
+    if (!this.wake.isActive || this.wake.currentPhase !== 'spotting') return;
+    if (useChat.getState().isSending) return;
+    Log.info('voice', `wake: ${keyword}`);
+    speech.stop();
+    useChat.getState().ping();
+    this.playChime(true);
+    this.wake.beginCommand();
+  }
+
   setHandsFree(on: boolean): void {
     useVoice.getState().setHandsFree(on);
     useSettings.getState().update({ voice: { handsFree: on } });
@@ -113,6 +204,11 @@ class VoiceController {
   async start(auto = false): Promise<void> {
     const voice = useVoice.getState();
     if (voice.phase !== 'off') return;
+    if (this.wake.isActive) {
+      // The always-on listener owns the microphone: open a command capture on it instead.
+      if (!auto) this.onWakeDetected('manuel');
+      return;
+    }
     const settings = useSettings.getState().settings.voice;
     const seq = ++this.startSeq;
     voice.setError(null);
@@ -218,7 +314,7 @@ class VoiceController {
     }
   }
 
-  private async transcribe(wav: WavResult): Promise<void> {
+  private async transcribe(wav: WavResult, fromWake = false): Promise<void> {
     const voice = useVoice.getState();
     voice.setPhase('transcribing');
     useChat.getState().setHud('thinking');
@@ -229,7 +325,8 @@ class VoiceController {
       voice.setTranscript(text);
       voice.setPhase('off');
       Log.info('voice', `transcript (${wav.durationSec.toFixed(1)}s): ${text}`);
-      if (voice.handsFree && settings.wakeWordEnabled && Date.now() > this.attentionUntil) {
+      const filterByTranscript = settings.wakeMode === 'transcript' || (settings.wakeMode === 'off' && settings.wakeWordEnabled);
+      if (!fromWake && voice.handsFree && filterByTranscript && Date.now() > this.attentionUntil) {
         const { matched, rest } = matchWakeWord(text, settings.wakeWord || 'jarvis');
         if (!matched) {
           Log.debug('voice', 'utterance ignored: no wake word');

src/services/voice/wakeListener.ts

@@ -0,0 +1,216 @@
+/**
+ * Always-on microphone for the keyword spotter. Streams 16 kHz PCM to the main process
+ * (which feeds the sherpa-onnx spotter); once the wake word is detected it captures the
+ * following utterance with the energy VAD and hands back a WAV, then resumes spotting.
+ * One microphone stream, no re-opening between phrases.
+ */
+import { bridge } from '../../lib/bridge';
+import { Log } from '../../lib/log';
+import { audioBus } from './audioBus';
+import { DEFAULT_VAD, EnergyVad, type VadOptions } from './vad';
+import { buildWav16k, rms, type WavResult } from './wav';
+
+const WORKLET_SOURCE = `
+class EveFlowWakeProcessor extends AudioWorkletProcessor {
+  constructor() { super(); this.buffer = new Float32Array(2048); this.offset = 0; }
+  process(inputs) {
+    const channel = inputs[0] && inputs[0][0];
+    if (!channel) return true;
+    let i = 0;
+    while (i < channel.length) {
+      const n = Math.min(channel.length - i, this.buffer.length - this.offset);
+      this.buffer.set(channel.subarray(i, i + n), this.offset);
+      this.offset += n; i += n;
+      if (this.offset === this.buffer.length) {
+        this.port.postMessage(this.buffer, [this.buffer.buffer]);
+        this.buffer = new Float32Array(2048); this.offset = 0;
+      }
+    }
+    return true;
+  }
+}
+registerProcessor('eveflow-wake', EveFlowWakeProcessor);
+`;
+
+export type WakePhase = 'off' | 'spotting' | 'command' | 'speech';
+
+export interface WakeCallbacks {
+  onPhase: (phase: WakePhase) => void;
+  onLevel?: (level: number) => void;
+  onUtterance: (wav: WavResult) => void;
+  onNoSpeech: () => void;
+  onError: (message: string) => void;
+}
+
+let workletUrl: string | null = null;
+
+export class WakeListener {
+  private ctx: AudioContext | null = null;
+  private stream: MediaStream | null = null;
+  private node: AudioWorkletNode | null = null;
+  private phase: WakePhase = 'off';
+  private vad: EnergyVad | null = null;
+  private chunks: Float32Array[] = [];
+  private totalSamples = 0;
+  private sampleRate = 16_000;
+  private pending: Float32Array[] = [];
+  private pendingSamples = 0;
+  private callbacks: WakeCallbacks | null = null;
+  private vadOptions: Partial<VadOptions> = {};
+  private noSpeechTimer: ReturnType<typeof setTimeout> | null = null;
+
+  get isActive(): boolean {
+    return this.phase !== 'off';
+  }
+
+  get currentPhase(): WakePhase {
+    return this.phase;
+  }
+
+  async start(options: { deviceId?: string; vad?: Partial<VadOptions>; callbacks: WakeCallbacks }): Promise<void> {
+    if (this.phase !== 'off') return;
+    this.callbacks = options.callbacks;
+    this.vadOptions = options.vad ?? {};
+    this.stream = await navigator.mediaDevices.getUserMedia({
+      audio: {
+        deviceId: options.deviceId ? { exact: options.deviceId } : undefined,
+        echoCancellation: true,
+        noiseSuppression: true,
+        autoGainControl: true,
+        channelCount: 1
+      }
+    });
+    try {
+      this.ctx = new AudioContext({ sampleRate: 16_000, latencyHint: 'playback' });
+    } catch {
+      this.ctx = new AudioContext({ latencyHint: 'playback' });
+    }
+    if (this.ctx.state === 'suspended') await this.ctx.resume().catch(() => undefined);
+    this.sampleRate = this.ctx.sampleRate;
+    const source = this.ctx.createMediaStreamSource(this.stream);
+    const analyser = this.ctx.createAnalyser();
+    analyser.fftSize = 512;
+    source.connect(analyser);
+    audioBus.setInputAnalyser(analyser);
+    if (!workletUrl) workletUrl = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'application/javascript' }));
+    await this.ctx.audioWorklet.addModule(workletUrl);
+    this.node = new AudioWorkletNode(this.ctx, 'eveflow-wake', { numberOfInputs: 1, numberOfOutputs: 0, channelCount: 1 });
+    this.node.port.onmessage = (event: MessageEvent<Float32Array>) => this.onSamples(event.data);
+    source.connect(this.node);
+    this.setPhase('spotting');
+    Log.info('wake', `listener started (${this.sampleRate} Hz)`);
+  }
+
+  /** Called by the controller when the main process reports the wake word (or on manual trigger). */
+  beginCommand(): void {
+    if (this.phase === 'off' || this.phase === 'command' || this.phase === 'speech') return;
+    this.vad = new EnergyVad({ ...DEFAULT_VAD, ...this.vadOptions, noSpeechTimeoutMs: Number.POSITIVE_INFINITY });
+    this.chunks = [];
+    this.totalSamples = 0;
+    this.setPhase('command');
+    if (this.noSpeechTimer) clearTimeout(this.noSpeechTimer);
+    this.noSpeechTimer = setTimeout(() => {
+      if (this.phase === 'command') {
+        this.resumeSpotting();
+        this.callbacks?.onNoSpeech();
+      }
+    }, 8000);
+  }
+
+  /** Abort a command capture and go back to spotting. */
+  cancelCommand(): void {
+    if (this.phase === 'command' || this.phase === 'speech') this.resumeSpotting();
+  }
+
+  stop(): void {
+    if (this.phase === 'off') return;
+    if (this.noSpeechTimer) clearTimeout(this.noSpeechTimer);
+    this.noSpeechTimer = null;
+    audioBus.setInputAnalyser(null);
+    if (this.node) {
+      this.node.port.onmessage = null;
+      this.node.disconnect();
+      this.node = null;
+    }
+    this.stream?.getTracks().forEach((t) => t.stop());
+    this.stream = null;
+    void this.ctx?.close().catch(() => undefined);
+    this.ctx = null;
+    this.chunks = [];
+    this.pending = [];
+    this.pendingSamples = 0;
+    this.setPhase('off');
+    Log.info('wake', 'listener stopped');
+  }
+
+  private setPhase(phase: WakePhase): void {
+    this.phase = phase;
+    this.callbacks?.onPhase(phase);
+  }
+
+  private resumeSpotting(): void {
+    if (this.noSpeechTimer) clearTimeout(this.noSpeechTimer);
+    this.noSpeechTimer = null;
+    this.vad = null;
+    this.chunks = [];
+    this.totalSamples = 0;
+    this.setPhase('spotting');
+  }
+
+  private onSamples(samples: Float32Array): void {
+    if (this.phase === 'off') return;
+    const level = rms(samples);
+    this.callbacks?.onLevel?.(Math.min(1, level * 6));
+
+    if (this.phase === 'spotting') {
+      // Batch ~256 ms of audio per IPC message for the keyword spotter.
+      this.pending.push(samples);
+      this.pendingSamples += samples.length;
+      if (this.pendingSamples >= this.sampleRate * 0.25) this.flushToSpotter();
+      return;
+    }
+
+    // command / speech: collect the utterance
+    this.chunks.push(samples);
+    this.totalSamples += samples.length;
+    if (this.vad && this.phase === 'command') {
+      // keep only a short pre-roll before speech starts
+      while (this.chunks.length > 1 && this.totalSamples - this.chunks[0].length > this.sampleRate * 0.4) {
+        this.totalSamples -= this.chunks.shift()!.length;
+      }
+    }
+    if (!this.vad) return;
+    const signal = this.vad.feed(level, (samples.length / this.sampleRate) * 1000);
+    if (signal === 'speech-start') {
+      this.setPhase('speech');
+      if (this.noSpeechTimer) clearTimeout(this.noSpeechTimer);
+      this.noSpeechTimer = null;
+    } else if (signal === 'speech-end' || signal === 'max-length') {
+      const wav = buildWav16k(this.chunks, this.sampleRate);
+      this.resumeSpotting();
+      if (wav.durationSec >= 0.25) this.callbacks?.onUtterance(wav);
+      else this.callbacks?.onNoSpeech();
+    } else if (signal === 'too-short') {
+      this.resumeSpotting();
+      this.callbacks?.onNoSpeech();
+    }
+  }
+
+  private flushToSpotter(): void {
+    const api = bridge();
+    if (!api) return;
+    const total = this.pendingSamples;
+    const merged = new Int16Array(total);
+    let offset = 0;
+    for (const chunk of this.pending) {
+      for (let i = 0; i < chunk.length; i++) {
+        const s = Math.max(-1, Math.min(1, chunk[i]));
+        merged[offset + i] = s < 0 ? s * 0x8000 : s * 0x7fff;
+      }
+      offset += chunk.length;
+    }
+    this.pending = [];
+    this.pendingSamples = 0;
+    api.voice.kwsAudio(new Uint8Array(merged.buffer), this.sampleRate);
+  }
+}