mirror of
https://github.com/R0m1k3/EveFlow.git
synced 2026-10-11 17:29:03 +02:00
LogiFlowandClaude
af8bdc5e9d
Écoute permanente - Silero VAD dans le worker sherpa-onnx (vad.start/audio/stop) ; le renderer envoie des trames de 128 ms pendant la commande et reçoit le segment WAV complet (speech-start / segment). Repli sur le VAD énergétique si le modèle n'est pas installé ou si l'option est désactivée. - Modèle « silero-vad » (0,6 Mo) dans le catalogue, section « Fin de phrase » dans Modèles locaux, réglage dans Paramètres → Micro. Vision d'écran - IPC system:screen-capture (desktopCapturer, JPEG 1600 px) ; bouton dans la barre de commande ; « regarde mon écran… » joint la capture à la question envoyée à Hermes (transport chat completions pour les images). Actions locales - IPC system:action à liste blanche : verrouiller la session, touches média (volume, mute, lecture, piste), ouvrir une application connue ou une URL http(s), presse-papiers, recherche de fichiers dans les dossiers utilisateur. - Routeur d'intentions FR/EN exécuté avant Hermes (« coupe le son », « ouvre Spotify », « verrouille la session »…), résultat affiché et lu ; désactivable. Correctif - Chat completions : les premiers fragments SSE arrivés avant l'événement de démarrage étaient perdus (premier mot manquant) ; ils sont maintenant rejoués. Tests : 26 tests unitaires (intentions locales ajoutées), e2e Electron sous Xvfb avec les vrais modèles (Silero : un segment par phrase, 6,7 s d'audio traités en 190 ms ; capture 109 ko reçue par Hermes ; intention locale traitée sans Hermes). Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y Co-authored-by: Claude <noreply@anthropic.com>
126 lines
2.9 KiB
TypeScript
126 lines
2.9 KiB
TypeScript
/** Local voice engine contract (sherpa-onnx in a utility process). Shared by main and renderer. */
|
|
|
|
export type VoiceModelKind = 'stt' | 'tts' | 'kws' | 'vad';
|
|
export type VoiceEngineKind = 'whisper' | 'sense-voice' | 'nemo-transducer' | 'kokoro' | 'piper' | 'kws-transducer' | 'silero';
|
|
|
|
export interface VoiceSpeaker {
|
|
id: number;
|
|
name: string;
|
|
lang: string;
|
|
}
|
|
|
|
export interface VoiceModelSpec {
|
|
id: string;
|
|
kind: VoiceModelKind;
|
|
engine: VoiceEngineKind;
|
|
name: string;
|
|
description: string;
|
|
languages: string[];
|
|
sizeMb: number;
|
|
url: string;
|
|
/** Folder created by the archive (files are referenced relative to it). */
|
|
dir: string;
|
|
/** Files that must exist once installed. */
|
|
files: string[];
|
|
speakers?: VoiceSpeaker[];
|
|
sampleRate?: number;
|
|
recommended?: boolean;
|
|
}
|
|
|
|
export interface VoiceModelStatus extends VoiceModelSpec {
|
|
installed: boolean;
|
|
downloading: boolean;
|
|
installedBytes: number;
|
|
}
|
|
|
|
export interface VoiceDownloadProgress {
|
|
id: string;
|
|
phase: 'download' | 'extract' | 'done' | 'error' | 'cancelled';
|
|
received: number;
|
|
total: number;
|
|
percent: number;
|
|
message?: string;
|
|
}
|
|
|
|
export interface TranscribeRequest {
|
|
modelId: string;
|
|
wav: Uint8Array;
|
|
language: string; // 'fr', 'en', 'auto'
|
|
}
|
|
|
|
export interface TranscribeResult {
|
|
text: string;
|
|
language?: string;
|
|
durationMs: number;
|
|
audioSec: number;
|
|
}
|
|
|
|
export interface SynthesizeRequest {
|
|
modelId: string;
|
|
text: string;
|
|
speaker: number;
|
|
speed: number;
|
|
}
|
|
|
|
export interface SynthesizeResult {
|
|
wav: Uint8Array;
|
|
sampleRate: number;
|
|
durationMs: number;
|
|
audioSec: number;
|
|
}
|
|
|
|
export interface KwsStartRequest {
|
|
modelId: string;
|
|
/** Wake phrases in plain text (e.g. "jarvis", "hey jarvis"). */
|
|
keywords: string[];
|
|
/** 1 (strict) .. 5 (eager) */
|
|
sensitivity: number;
|
|
}
|
|
|
|
export interface KwsDetection {
|
|
keyword: string;
|
|
at: number;
|
|
}
|
|
|
|
export interface VadStartRequest {
|
|
modelId: string;
|
|
/** Silence that ends an utterance, in ms. */
|
|
silenceMs: number;
|
|
/** Detection threshold 0..1 (0.5 default). */
|
|
threshold: number;
|
|
maxUtteranceSec: number;
|
|
}
|
|
|
|
export type VadEvent =
|
|
| { type: 'speech-start' }
|
|
| { type: 'segment'; wav: Uint8Array; durationSec: number }
|
|
| { type: 'error'; message: string };
|
|
|
|
export interface VoiceEngineStatus {
|
|
available: boolean;
|
|
error?: string;
|
|
version?: string;
|
|
loaded: string[];
|
|
modelsDir: string;
|
|
}
|
|
|
|
export const VOICE_IPC = {
|
|
status: 'voice:status',
|
|
modelsList: 'voice:models:list',
|
|
modelsDownload: 'voice:models:download',
|
|
modelsCancel: 'voice:models:cancel',
|
|
modelsRemove: 'voice:models:remove',
|
|
modelsProgress: 'voice:models:progress',
|
|
transcribe: 'voice:transcribe',
|
|
synthesize: 'voice:synthesize',
|
|
unload: 'voice:unload',
|
|
kwsStart: 'voice:kws:start',
|
|
kwsStop: 'voice:kws:stop',
|
|
kwsAudio: 'voice:kws:audio',
|
|
kwsDetected: 'voice:kws:detected',
|
|
vadStart: 'voice:vad:start',
|
|
vadStop: 'voice:vad:stop',
|
|
vadAudio: 'voice:vad:audio',
|
|
vadEvent: 'voice:vad:event'
|
|
} as const;
|