feat: fin de phrase Silero, vision d'écran et actions locales (v2.3.0) (#13)

Écoute permanente
- Silero VAD dans le worker sherpa-onnx (vad.start/audio/stop) ; le renderer
  envoie des trames de 128 ms pendant la commande et reçoit le segment WAV
  complet (speech-start / segment). Repli sur le VAD énergétique si le modèle
  n'est pas installé ou si l'option est désactivée.
- Modèle « silero-vad » (0,6 Mo) dans le catalogue, section « Fin de phrase »
  dans Modèles locaux, réglage dans Paramètres → Micro.

Vision d'écran
- IPC system:screen-capture (desktopCapturer, JPEG 1600 px) ; bouton dans la
  barre de commande ; « regarde mon écran… » joint la capture à la question
  envoyée à Hermes (transport chat completions pour les images).

Actions locales
- IPC system:action à liste blanche : verrouiller la session, touches média
  (volume, mute, lecture, piste), ouvrir une application connue ou une URL
  http(s), presse-papiers, recherche de fichiers dans les dossiers utilisateur.
- Routeur d'intentions FR/EN exécuté avant Hermes (« coupe le son », « ouvre
  Spotify », « verrouille la session »…), résultat affiché et lu ; désactivable.

Correctif
- Chat completions : les premiers fragments SSE arrivés avant l'événement de
  démarrage étaient perdus (premier mot manquant) ; ils sont maintenant rejoués.

Tests : 26 tests unitaires (intentions locales ajoutées), e2e Electron sous
Xvfb avec les vrais modèles (Silero : un segment par phrase, 6,7 s d'audio
traités en 190 ms ; capture 109 ko reçue par Hermes ; intention locale
traitée sans Hermes).


Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y

Co-authored-by: Claude <noreply@anthropic.com>

src/services/voice/voiceController.ts

@@ -113,8 +113,21 @@ class VoiceController {
       this.unsubscribeKws?.();
       this.unsubscribeKws = api.voice.onKwsDetected((d) => this.onWakeDetected(d.keyword));
       const sensitivity = Math.min(5, Math.max(1, Math.round(settings.sensitivity))) as 1 | 2 | 3 | 4 | 5;
+      let neuralVad = false;
+      if (settings.neuralVad) {
+        try {
+          const installed = (await api.voice.listModels()).some((m) => m.id === 'silero-vad' && m.installed);
+          if (installed) {
+            await api.voice.vadStart({ modelId: 'silero-vad', silenceMs: Math.max(300, Math.min(1500, settings.silenceMs - 200)), threshold: 0.5, maxUtteranceSec: 25 });
+            neuralVad = true;
+          }
+        } catch (err) {
+          Log.warn('voice', `silero unavailable, energy VAD fallback: ${(err as Error).message}`);
+        }
+      }
       await this.wake.start({
         deviceId: settings.micDeviceId || undefined,
+        neuralVad,
         vad: { silenceMs: settings.silenceMs, speechRatio: SENSITIVITY_RATIO[sensitivity], minRms: SENSITIVITY_MIN_RMS[sensitivity] },
         callbacks: {
           onPhase: (phase) => {
@@ -141,7 +154,8 @@ class VoiceController {
         }
       });
       voice.setWake('spotting', result.accepted.map((k) => k.replace(/_/g, ' ')));
-      Log.info('voice', `wake mode on: ${result.accepted.join(', ')}${result.rejected.length ? ` (rejetés : ${result.rejected.join(', ')})` : ''}`);
+      voice.setNeuralVad(neuralVad);
+      Log.info('voice', `wake mode on: ${result.accepted.join(', ')}${result.rejected.length ? ` (rejetés : ${result.rejected.join(', ')})` : ''}${neuralVad ? ' · Silero' : ''}`);
     } catch (err) {
       const message = (err as Error).message;
       voice.setWake('error');
@@ -149,6 +163,7 @@ class VoiceController {
       useChat.getState().setError(`Mot d’activation : ${message}`);
       Log.error('voice', `wake mode failed: ${message}`);
       await api.voice.kwsStop().catch(() => undefined);
+      await api.voice.vadStop().catch(() => undefined);
     }
   }
 
@@ -157,7 +172,10 @@ class VoiceController {
     this.unsubscribeKws = null;
     this.wake.stop();
     useVoice.getState().setWake('off');
-    await bridge()?.voice.kwsStop().catch(() => undefined);
+    useVoice.getState().setNeuralVad(false);
+    const api = bridge();
+    await api?.voice.kwsStop().catch(() => undefined);
+    await api?.voice.vadStop().catch(() => undefined);
   }
 
   /** Restart spotting with the current settings (wake word, sensitivity, microphone). */

src/services/voice/wakeListener.ts

@@ -9,6 +9,7 @@ import { Log } from '../../lib/log';
 import { audioBus } from './audioBus';
 import { DEFAULT_VAD, EnergyVad, type VadOptions } from './vad';
 import { buildWav16k, rms, type WavResult } from './wav';
+import type { VadEvent } from '../../../shared/voice';
 
 const WORKLET_SOURCE = `
 class EveFlowWakeProcessor extends AudioWorkletProcessor {
@@ -58,6 +59,11 @@ export class WakeListener {
   private callbacks: WakeCallbacks | null = null;
   private vadOptions: Partial<VadOptions> = {};
   private noSpeechTimer: ReturnType<typeof setTimeout> | null = null;
+  /** Neural end-of-speech (Silero in the main process) instead of the energy VAD. */
+  private neural = false;
+  private unsubscribeVad: (() => void) | null = null;
+  private vadPending: Float32Array[] = [];
+  private vadPendingSamples = 0;
 
   get isActive(): boolean {
     return this.phase !== 'off';
@@ -67,10 +73,15 @@ export class WakeListener {
     return this.phase;
   }
 
-  async start(options: { deviceId?: string; vad?: Partial<VadOptions>; callbacks: WakeCallbacks }): Promise<void> {
+  async start(options: { deviceId?: string; vad?: Partial<VadOptions>; neuralVad?: boolean; callbacks: WakeCallbacks }): Promise<void> {
     if (this.phase !== 'off') return;
     this.callbacks = options.callbacks;
     this.vadOptions = options.vad ?? {};
+    this.neural = !!options.neuralVad;
+    if (this.neural) {
+      this.unsubscribeVad?.();
+      this.unsubscribeVad = bridge()?.voice.onVadEvent((event) => this.onVadEvent(event)) ?? null;
+    }
     this.stream = await navigator.mediaDevices.getUserMedia({
       audio: {
         deviceId: options.deviceId ? { exact: options.deviceId } : undefined,
@@ -98,13 +109,34 @@ export class WakeListener {
     this.node.port.onmessage = (event: MessageEvent<Float32Array>) => this.onSamples(event.data);
     source.connect(this.node);
     this.setPhase('spotting');
-    Log.info('wake', `listener started (${this.sampleRate} Hz)`);
+    Log.info('wake', `listener started (${this.sampleRate} Hz, fin de phrase ${this.neural ? 'Silero' : 'énergie'})`);
+  }
+
+  private onVadEvent(event: VadEvent): void {
+    if (!this.neural || (this.phase !== 'command' && this.phase !== 'speech')) return;
+    if (event.type === 'speech-start') {
+      if (this.phase === 'command') {
+        this.setPhase('speech');
+        if (this.noSpeechTimer) clearTimeout(this.noSpeechTimer);
+        this.noSpeechTimer = null;
+      }
+    } else if (event.type === 'segment') {
+      const wav: WavResult = { bytes: event.wav, sampleRate: 16_000, durationSec: event.durationSec };
+      this.resumeSpotting();
+      if (wav.durationSec >= 0.25) this.callbacks?.onUtterance(wav);
+      else this.callbacks?.onNoSpeech();
+    } else if (event.type === 'error') {
+      this.callbacks?.onError(event.message);
+      this.resumeSpotting();
+    }
   }
 
   /** Called by the controller when the main process reports the wake word (or on manual trigger). */
   beginCommand(): void {
     if (this.phase === 'off' || this.phase === 'command' || this.phase === 'speech') return;
-    this.vad = new EnergyVad({ ...DEFAULT_VAD, ...this.vadOptions, noSpeechTimeoutMs: Number.POSITIVE_INFINITY });
+    this.vad = this.neural ? null : new EnergyVad({ ...DEFAULT_VAD, ...this.vadOptions, noSpeechTimeoutMs: Number.POSITIVE_INFINITY });
+    this.vadPending = [];
+    this.vadPendingSamples = 0;
     this.chunks = [];
     this.totalSamples = 0;
     this.setPhase('command');
@@ -126,6 +158,8 @@ export class WakeListener {
     if (this.phase === 'off') return;
     if (this.noSpeechTimer) clearTimeout(this.noSpeechTimer);
     this.noSpeechTimer = null;
+    this.unsubscribeVad?.();
+    this.unsubscribeVad = null;
     audioBus.setInputAnalyser(null);
     if (this.node) {
       this.node.port.onmessage = null;
@@ -139,6 +173,8 @@ export class WakeListener {
     this.chunks = [];
     this.pending = [];
     this.pendingSamples = 0;
+    this.vadPending = [];
+    this.vadPendingSamples = 0;
     this.setPhase('off');
     Log.info('wake', 'listener stopped');
   }
@@ -170,6 +206,14 @@ export class WakeListener {
       return;
     }
 
+    if (this.neural) {
+      // command / speech with Silero: stream ~128 ms frames to the main process, which returns the segment.
+      this.vadPending.push(samples);
+      this.vadPendingSamples += samples.length;
+      if (this.vadPendingSamples >= this.sampleRate * 0.128) this.flushToVad();
+      return;
+    }
+
     // command / speech: collect the utterance
     this.chunks.push(samples);
     this.totalSamples += samples.length;
@@ -196,21 +240,34 @@ export class WakeListener {
     }
   }
 
-  private flushToSpotter(): void {
-    const api = bridge();
-    if (!api) return;
-    const total = this.pendingSamples;
+  private static toInt16(chunks: Float32Array[], total: number): Uint8Array {
     const merged = new Int16Array(total);
     let offset = 0;
-    for (const chunk of this.pending) {
+    for (const chunk of chunks) {
       for (let i = 0; i < chunk.length; i++) {
         const s = Math.max(-1, Math.min(1, chunk[i]));
         merged[offset + i] = s < 0 ? s * 0x8000 : s * 0x7fff;
       }
       offset += chunk.length;
     }
+    return new Uint8Array(merged.buffer);
+  }
+
+  private flushToSpotter(): void {
+    const api = bridge();
+    if (!api) return;
+    const bytes = WakeListener.toInt16(this.pending, this.pendingSamples);
     this.pending = [];
     this.pendingSamples = 0;
-    api.voice.kwsAudio(new Uint8Array(merged.buffer), this.sampleRate);
+    api.voice.kwsAudio(bytes, this.sampleRate);
+  }
+
+  private flushToVad(): void {
+    const api = bridge();
+    if (!api) return;
+    const bytes = WakeListener.toInt16(this.vadPending, this.vadPendingSamples);
+    this.vadPending = [];
+    this.vadPendingSamples = 0;
+    api.voice.vadAudio(bytes, this.sampleRate);
   }
 }