feat: always-on wake word with sherpa-onnx keyword spotting (v2.2.0)

- Catalog: 3.3 MB zipformer keyword-spotting model (kws-en).
- shared/keywords: BPE encoding of wake phrases (SentencePiece table for common words,
  greedy longest-match fallback over the model vocabulary) and keywords file builder.
- Worker: KeywordSpotter stream fed with 16-bit PCM, detections pushed as unsolicited
  messages; engine derives the keywords file, maps sensitivity to threshold/score,
  forwards detections to the renderer and re-arms after a worker restart.
- Renderer: WakeListener keeps one microphone stream, batches 256 ms frames to the
  spotter and captures the command on the same stream after detection (pre-roll, VAD),
  then resumes spotting; the wake word also interrupts speech. Settings: wake mode
  (off / always-on / transcript filter), keyword, sensitivity, status and one-click
  model download; HUD caption shows the active keyword.
- Validated: detection in the worker (fork) and through the real Electron IPC path on
  Kokoro audio, no false positive on an English recording; 23 unit tests.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y

electron/preload.ts

@@ -14,7 +14,7 @@ import {
   type WindowMode
 } from '../shared/ipc';
 import type { EveFlowBridge, Unsubscribe } from '../shared/bridge';
-import { VOICE_IPC, type SynthesizeRequest, type SynthesizeResult, type TranscribeRequest, type TranscribeResult, type VoiceDownloadProgress, type VoiceEngineStatus, type VoiceModelStatus } from '../shared/voice';
+import { VOICE_IPC, type KwsDetection, type KwsStartRequest, type SynthesizeRequest, type SynthesizeResult, type TranscribeRequest, type TranscribeResult, type VoiceDownloadProgress, type VoiceEngineStatus, type VoiceModelStatus } from '../shared/voice';
 
 function subscribe<T>(channel: string, callback: (payload: T) => void): Unsubscribe {
   const listener = (_event: Electron.IpcRendererEvent, payload: T) => callback(payload);
@@ -69,7 +69,11 @@ const api: EveFlowBridge = {
     onProgress: (cb: (progress: VoiceDownloadProgress) => void) => subscribe<VoiceDownloadProgress>(VOICE_IPC.modelsProgress, cb),
     transcribe: (req: TranscribeRequest) => ipcRenderer.invoke(VOICE_IPC.transcribe, req) as Promise<TranscribeResult>,
     synthesize: (req: SynthesizeRequest) => ipcRenderer.invoke(VOICE_IPC.synthesize, req) as Promise<SynthesizeResult>,
-    unload: (id?: string) => ipcRenderer.invoke(VOICE_IPC.unload, id) as Promise<unknown>
+    unload: (id?: string) => ipcRenderer.invoke(VOICE_IPC.unload, id) as Promise<unknown>,
+    kwsStart: (req: KwsStartRequest) => ipcRenderer.invoke(VOICE_IPC.kwsStart, req) as Promise<{ accepted: string[]; rejected: string[] }>,
+    kwsStop: () => ipcRenderer.invoke(VOICE_IPC.kwsStop) as Promise<void>,
+    kwsAudio: (pcm: Uint8Array, sampleRate: number) => ipcRenderer.send(VOICE_IPC.kwsAudio, pcm, sampleRate),
+    onKwsDetected: (cb: (detection: KwsDetection) => void) => subscribe<KwsDetection>(VOICE_IPC.kwsDetected, cb)
   }
 };
 

electron/voice/catalog.ts

@@ -1,6 +1,7 @@
 import type { VoiceModelSpec, VoiceSpeaker } from '../../shared/voice';
 
 const ASR = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models';
+const KWS = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/kws-models';
 const TTS = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models';
 
 const KOKORO_SPEAKERS: VoiceSpeaker[] = [
@@ -67,6 +68,24 @@ export const VOICE_CATALOG: VoiceModelSpec[] = [
     dir: 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17',
     files: ['model.int8.onnx', 'tokens.txt']
   },
+  {
+    id: 'kws-en',
+    kind: 'kws',
+    engine: 'kws-transducer',
+    name: 'Détecteur de mot-clé (zipformer, 3 Mo)',
+    description: 'Écoute permanente du mot d’activation, quasi gratuite en CPU. Mot-clé libre (« jarvis », « hey jarvis »…).',
+    languages: ['en', 'fr'],
+    sizeMb: 4,
+    url: `${KWS}/sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01.tar.bz2`,
+    dir: 'sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01',
+    files: [
+      'encoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx',
+      'decoder-epoch-12-avg-2-chunk-16-left-64.int8.onnx',
+      'joiner-epoch-12-avg-2-chunk-16-left-64.int8.onnx',
+      'tokens.txt'
+    ],
+    recommended: true
+  },
   {
     id: 'kokoro-v1',
     kind: 'tts',

electron/voice/engine.ts

@@ -2,13 +2,21 @@
  * Host side of the voice worker: spawns the utility process on demand, correlates
  * requests and responses, restarts the worker if it crashes.
  */
-import { utilityProcess, type UtilityProcess } from 'electron';
+import { utilityProcess, type UtilityProcess, type WebContents } from 'electron';
+import { createHash } from 'node:crypto';
+import fs from 'node:fs';
 import path from 'node:path';
-import type { SynthesizeRequest, SynthesizeResult, TranscribeRequest, TranscribeResult, VoiceEngineStatus } from '../../shared/voice';
+import { VOICE_IPC, type KwsDetection, type KwsStartRequest, type SynthesizeRequest, type SynthesizeResult, type TranscribeRequest, type TranscribeResult, type VoiceEngineStatus } from '../../shared/voice';
+import { buildKeywordsFile, parseTokens } from '../../shared/keywords';
 import { findModel } from './catalog';
 import { isInstalled, modelDir, modelsDir } from './models';
 import { log } from '../logger';
 
+/** Renderer that receives keyword detections while spotting is active. */
+let kwsSubscriber: WebContents | null = null;
+let kwsActive = false;
+let kwsRequest: KwsStartRequest | null = null;
+
 interface Pending {
   resolve: (value: unknown) => void;
   reject: (err: Error) => void;
@@ -34,7 +42,15 @@ function spawn(): UtilityProcess {
   child = proc;
   proc.stdout?.on('data', (d: Buffer) => log('DEBUG', 'voice-worker', d.toString().trim()));
   proc.stderr?.on('data', (d: Buffer) => log('WARN', 'voice-worker', d.toString().trim()));
-  proc.on('message', (msg: { id: number; ok: boolean; result?: unknown; error?: string }) => {
+  proc.on('message', (msg: { id?: number; type?: string; ok?: boolean; result?: unknown; error?: string; keyword?: string; at?: number }) => {
+    if (msg.type === 'kws.detected') {
+      if (kwsSubscriber && !kwsSubscriber.isDestroyed()) {
+        kwsSubscriber.send(VOICE_IPC.kwsDetected, { keyword: msg.keyword ?? '', at: msg.at ?? Date.now() } satisfies KwsDetection);
+      }
+      log('INFO', 'voice', `wake word detected: ${msg.keyword}`);
+      return;
+    }
+    if (typeof msg.id !== 'number') return;
     const p = pending.get(msg.id);
     if (!p) return;
     pending.delete(msg.id);
@@ -48,6 +64,8 @@ function spawn(): UtilityProcess {
     if (child !== proc) return;
     child = null;
     rejectAll('Le moteur vocal local s’est arrêté de façon inattendue.');
+    // Keyword spotting survives a worker restart: re-arm on the next audio frame.
+    if (kwsActive) kwsArmed = false;
   });
   log('INFO', 'voice', 'voice worker started');
   return proc;
@@ -108,6 +126,60 @@ export async function synthesize(req: SynthesizeRequest): Promise<SynthesizeResu
   return { ...result, wav: new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.byteLength) };
 }
 
+let kwsArmed = false;
+const SENSITIVITY_THRESHOLD: Record<number, number> = { 1: 0.45, 2: 0.35, 3: 0.25, 4: 0.18, 5: 0.12 };
+const SENSITIVITY_SCORE: Record<number, number> = { 1: 1.0, 2: 1.0, 3: 1.2, 4: 1.5, 5: 2.0 };
+
+/** Start keyword spotting; the keywords file is derived from the phrases and the model vocabulary. */
+export async function kwsStart(req: KwsStartRequest, sender: WebContents): Promise<{ accepted: string[]; rejected: string[] }> {
+  const spec = findModel(req.modelId);
+  if (!spec || spec.kind !== 'kws') throw new Error('Modèle de détection introuvable');
+  if (!isInstalled(spec)) throw new Error('Détecteur de mot-clé non installé (Paramètres → Modèles locaux).');
+  const dir = modelDir(spec);
+  const vocab = parseTokens(fs.readFileSync(path.join(dir, 'tokens.txt'), 'utf8'));
+  const phrases = req.keywords.map((k) => String(k).slice(0, 40)).filter(Boolean).slice(0, 8);
+  const file = buildKeywordsFile(phrases, vocab);
+  if (file.accepted.length === 0) throw new Error(`Aucun mot d’activation encodable : ${file.rejected.join(', ')}`);
+  const hash = createHash('sha1').update(file.content).digest('hex').slice(0, 10);
+  const keywordsFile = path.join(modelsDir(), `keywords-${hash}.txt`);
+  fs.writeFileSync(keywordsFile, file.content, 'utf8');
+  const sensitivity = Math.min(5, Math.max(1, Math.round(req.sensitivity))) as 1 | 2 | 3 | 4 | 5;
+  kwsSubscriber = sender;
+  kwsRequest = req;
+  await request({ type: 'kws.start', model: { id: spec.id, engine: spec.engine, dir, files: spec.files }, keywordsFile, threshold: SENSITIVITY_THRESHOLD[sensitivity], score: SENSITIVITY_SCORE[sensitivity] }, 60_000);
+  kwsActive = true;
+  kwsArmed = true;
+  log('INFO', 'voice', `keyword spotting on: ${file.accepted.join(', ')} (threshold ${SENSITIVITY_THRESHOLD[sensitivity]})`);
+  return { accepted: file.accepted, rejected: file.rejected };
+}
+
+export async function kwsStop(): Promise<void> {
+  kwsActive = false;
+  kwsArmed = false;
+  kwsSubscriber = null;
+  if (child) await request({ type: 'kws.stop' }, 10_000).catch(() => undefined);
+}
+
+/** Feed 16-bit PCM from the renderer (fire-and-forget). */
+export function kwsFeed(pcm: Uint8Array, sampleRate: number): void {
+  if (!kwsActive) return;
+  const proc = spawn();
+  if (!kwsArmed) {
+    // Worker restarted: re-create the spotter before feeding audio.
+    if (kwsRequest && kwsSubscriber) {
+      kwsArmed = true;
+      kwsStart(kwsRequest, kwsSubscriber).catch((err) => log('WARN', 'voice', `kws re-arm failed: ${(err as Error).message}`));
+    }
+    return;
+  }
+  const b64 = Buffer.from(pcm.buffer, pcm.byteOffset, pcm.byteLength).toString('base64');
+  try {
+    proc.postMessage({ id: -1, type: 'kws.audio', pcm: b64, sampleRate });
+  } catch (err) {
+    log('WARN', 'voice', `kws feed failed: ${(err as Error).message}`);
+  }
+}
+
 /** Native model memory is only released deterministically by restarting the worker. */
 export function unload(_modelId?: string): Promise<unknown> {
   stopEngine();

electron/voice/ipc.ts

@@ -1,6 +1,6 @@
 import { ipcMain } from 'electron';
-import { VOICE_IPC, type SynthesizeRequest, type TranscribeRequest } from '../../shared/voice';
-import { engineStatus, synthesize, transcribe, unload } from './engine';
+import { VOICE_IPC, type KwsStartRequest, type SynthesizeRequest, type TranscribeRequest } from '../../shared/voice';
+import { engineStatus, kwsFeed, kwsStart, kwsStop, synthesize, transcribe, unload } from './engine';
 import { cancelDownload, downloadModel, listModels, removeModel } from './models';
 
 export function registerVoiceIpc(): void {
@@ -27,4 +27,13 @@ export function registerVoiceIpc(): void {
     return synthesize({ ...req, speaker: Number.isFinite(req.speaker) ? req.speaker : 0, speed: Number.isFinite(req.speed) ? req.speed : 1 });
   });
   ipcMain.handle(VOICE_IPC.unload, (_e, id?: string) => unload(id));
+  ipcMain.handle(VOICE_IPC.kwsStart, (event, req: KwsStartRequest) => {
+    if (!req || !Array.isArray(req.keywords) || typeof req.modelId !== 'string') throw new Error('Requête invalide');
+    return kwsStart({ ...req, keywords: req.keywords.filter((k) => typeof k === 'string'), sensitivity: Number(req.sensitivity) || 3 }, event.sender);
+  });
+  ipcMain.handle(VOICE_IPC.kwsStop, () => kwsStop());
+  ipcMain.on(VOICE_IPC.kwsAudio, (_e, pcm: unknown, sampleRate: unknown) => {
+    if (!(pcm instanceof Uint8Array) || pcm.byteLength === 0 || pcm.byteLength > 1024 * 1024) return;
+    kwsFeed(pcm, typeof sampleRate === 'number' && sampleRate > 0 ? sampleRate : 16000);
+  });
 }

electron/voice/worker.ts

@@ -18,7 +18,10 @@ type Request =
   | { id: number; type: 'status' }
   | { id: number; type: 'transcribe'; model: ModelRef; wav: Uint8Array | string; language: string }
   | { id: number; type: 'synthesize'; model: ModelRef; text: string; speaker: number; speed: number }
-  | { id: number; type: 'unload'; modelId?: string };
+  | { id: number; type: 'unload'; modelId?: string }
+  | { id: number; type: 'kws.start'; model: ModelRef; keywordsFile: string; threshold: number; score: number }
+  | { id: number; type: 'kws.audio'; pcm: string; sampleRate: number }
+  | { id: number; type: 'kws.stop' };
 
 type Response = { id: number; ok: true; result: unknown } | { id: number; ok: false; error: string };
 
@@ -29,6 +32,13 @@ type Sherpa = {
     decode: (s: unknown) => void;
     getResult: (s: unknown) => { text: string; lang?: string };
   };
+  KeywordSpotter: new (config: unknown) => {
+    createStream: () => KwsStream;
+    isReady: (s: KwsStream) => boolean;
+    decode: (s: KwsStream) => void;
+    reset: (s: KwsStream) => void;
+    getResult: (s: KwsStream) => { keyword?: string };
+  };
   OfflineTts: new (config: unknown) => {
     numSpeakers: number;
     sampleRate: number;
@@ -37,7 +47,10 @@ type Sherpa = {
   version: string;
 };
 
+type KwsStream = { acceptWaveform: (w: { sampleRate: number; samples: Float32Array }) => void };
 let sherpa: Sherpa | null = null;
+let kws: { spotter: InstanceType<Sherpa['KeywordSpotter']>; stream: KwsStream } | null = null;
+let notify: ((message: unknown) => void) | null = null;
 let loadError: string | null = null;
 
 function loadSherpa(): Sherpa {
@@ -225,13 +238,59 @@ function encodeWav(samples: Float32Array, sampleRate: number): Uint8Array {
   return new Uint8Array(buffer);
 }
 
+function startKws(model: ModelRef, keywordsFile: string, threshold: number, score: number): void {
+  const s = loadSherpa();
+  kws = null;
+  const p = (f: string) => path.join(model.dir, f);
+  const enc = model.files.find((f) => f.startsWith('encoder')) ?? 'encoder.int8.onnx';
+  const dec = model.files.find((f) => f.startsWith('decoder')) ?? 'decoder.int8.onnx';
+  const join = model.files.find((f) => f.startsWith('joiner')) ?? 'joiner.int8.onnx';
+  const spotter = new s.KeywordSpotter({
+    featConfig: { sampleRate: 16000, featureDim: 80 },
+    modelConfig: { transducer: { encoder: p(enc), decoder: p(dec), joiner: p(join) }, tokens: p('tokens.txt'), numThreads: 1, provider: 'cpu', debug: 0 },
+    maxActivePaths: 4,
+    numTrailingBlanks: 1,
+    keywordsScore: score,
+    keywordsThreshold: threshold,
+    keywordsFile
+  });
+  kws = { spotter, stream: spotter.createStream() };
+}
+
+function feedKws(pcmBase64: string, sampleRate: number): void {
+  if (!kws) return;
+  const bytes = Buffer.from(pcmBase64, 'base64');
+  const int16 = new Int16Array(bytes.buffer, bytes.byteOffset, Math.floor(bytes.byteLength / 2));
+  let samples: Float32Array = new Float32Array(int16.length);
+  for (let i = 0; i < int16.length; i++) samples[i] = int16[i] / 32768;
+  if (sampleRate !== 16000) samples = resampleTo16k(samples, sampleRate);
+  kws.stream.acceptWaveform({ sampleRate: 16000, samples });
+  while (kws.spotter.isReady(kws.stream)) {
+    kws.spotter.decode(kws.stream);
+    const result = kws.spotter.getResult(kws.stream);
+    if (result.keyword) {
+      kws.spotter.reset(kws.stream);
+      notify?.({ type: 'kws.detected', keyword: result.keyword, at: Date.now() });
+    }
+  }
+}
+
 // ── request handling ───────────────────────────────────────────────────────
 function handle(req: Request): unknown {
   switch (req.type) {
+    case 'kws.start':
+      startKws(req.model, req.keywordsFile, req.threshold, req.score);
+      return { ok: true };
+    case 'kws.audio':
+      feedKws(req.pcm, req.sampleRate);
+      return { ok: true };
+    case 'kws.stop':
+      kws = null;
+      return { ok: true };
     case 'status': {
       try {
         const s = loadSherpa();
-        return { available: true, version: s.version, loaded: [...recognizers.keys(), ...synthesizers.keys()] };
+        return { available: true, version: s.version, loaded: [...recognizers.keys(), ...synthesizers.keys(), ...(kws ? ['kws'] : [])] };
       } catch (err) {
         return { available: false, error: (err as Error).message, loaded: [] };
       }
@@ -271,6 +330,7 @@ function handle(req: Request): unknown {
       } else {
         recognizers.clear();
         synthesizers.clear();
+        kws = null;
       }
       return { ok: true };
     }
@@ -290,7 +350,9 @@ function respond(req: Request): Response {
 // Electron utility process transport, with a child_process fallback for tests.
 const parentPort = (process as unknown as { parentPort?: { on: (ev: 'message', cb: (e: { data: Request }) => void) => void; postMessage: (m: unknown) => void } }).parentPort;
 if (parentPort) {
+  notify = (m) => parentPort.postMessage(m);
   parentPort.on('message', (event) => parentPort.postMessage(respond(event.data)));
 } else if (process.send) {
+  notify = (m) => process.send!(m);
   process.on('message', (msg: Request) => process.send!(respond(msg)));
 }