mirror of
https://github.com/R0m1k3/EveFlow.git
synced 2026-10-16 19:58:07 +02:00
feat: local offline voice models (sherpa-onnx) with model manager and wake word (v2.1.0)
- electron/voice: model catalog (Whisper base/small/turbo, SenseVoice, Kokoro v1.0, Piper fr), streaming tar.bz2 downloader with progress, sherpa-onnx worker running in an Electron utilityProcess (transcribe / synthesize / status / unload), IPC + bridge. - Renderer: 'local' providers for STT and TTS, Settings → Modèles locaux (download, progress, delete, activate), speaker selection, hands-free wake word with a tolerant matcher (accents, punctuation, edit distance) and attention window after a bare wake word. - Packaging: native addon and worker unpacked from the asar; release 2.1.0. Validated on Linux: Kokoro (fr) → Whisper base round trip, Piper (fr) → Whisper round trip. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y
This commit is contained in:
22 files changed
+1361
-16
No files matched your search
@@ -10,6 +10,7 @@ import { useChat } from './state/chat';
|
||||
import { useHermes } from './state/hermes';
|
||||
import { useSettings } from './state/settings';
|
||||
import { useVoice } from './state/voice';
|
||||
import { useVoiceModels } from './state/voiceModels';
|
||||
import { ChatPanel } from './components/chat/ChatPanel';
|
||||
import { PendingRequests } from './components/chat/PendingRequests';
|
||||
import { CompactWidget } from './components/compact/CompactWidget';
|
||||
@@ -41,6 +42,8 @@ function useBoot(): boolean {
|
||||
Log.error('app', 'window.eveflow missing inside Electron');
|
||||
}
|
||||
if (api) {
|
||||
disposers.push(useVoiceModels.getState().subscribe());
|
||||
void useVoiceModels.getState().refresh();
|
||||
disposers.push(api.hermes.onPush(handlePush));
|
||||
disposers.push(
|
||||
api.hotkeys.on((event) => {
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
import { useEffect } from 'react';
|
||||
import { Download, Trash2, X, CheckCircle2, Cpu, Mic, Volume2, AlertTriangle, RefreshCw } from 'lucide-react';
|
||||
import type { VoiceModelStatus } from '../../../shared/voice';
|
||||
import { bridge } from '../../lib/bridge';
|
||||
import { useVoiceModels } from '../../state/voiceModels';
|
||||
import { useSettings } from '../../state/settings';
|
||||
|
||||
function formatMb(bytes: number): string {
|
||||
return bytes >= 1024 * 1024 * 1024 ? `${(bytes / 1024 ** 3).toFixed(2)} Go` : `${Math.round(bytes / 1024 / 1024)} Mo`;
|
||||
}
|
||||
|
||||
function ModelRow({ model }: { model: VoiceModelStatus }) {
|
||||
const progress = useVoiceModels((s) => s.progress[model.id]);
|
||||
const { download, cancel, remove } = useVoiceModels();
|
||||
const settings = useSettings((s) => s.settings);
|
||||
const update = useSettings((s) => s.update);
|
||||
const isActive = model.kind === 'stt' ? settings.voice.localModel === model.id : settings.speech.localModel === model.id;
|
||||
const busy = !!progress && (progress.phase === 'download' || progress.phase === 'extract');
|
||||
|
||||
const activate = () => {
|
||||
if (model.kind === 'stt') update({ voice: { localModel: model.id, provider: 'local' } });
|
||||
else update({ speech: { localModel: model.id, provider: 'local', localSpeaker: model.speakers?.[0]?.id ?? 0 } });
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="row-item" style={isActive && model.installed ? { borderColor: 'var(--accent)' } : undefined}>
|
||||
<div className="main">
|
||||
<div className="row wrap" style={{ gap: 6 }}>
|
||||
<span className="name">{model.name}</span>
|
||||
{model.recommended && <span className="status-pill ok">recommandé</span>}
|
||||
{model.installed && <span className="status-pill online">installé · {formatMb(model.installedBytes)}</span>}
|
||||
{!model.installed && !busy && <span className="status-pill">{model.sizeMb} Mo</span>}
|
||||
{isActive && model.installed && <span className="status-pill active">actif</span>}
|
||||
</div>
|
||||
<span className="desc">{model.description}</span>
|
||||
<span className="meta">langues : {model.languages.join(', ')}{model.speakers ? ` · ${model.speakers.length} voix` : ''}</span>
|
||||
{busy && (
|
||||
<div style={{ marginTop: 6 }}>
|
||||
<div className="level-meter"><div style={{ width: `${progress.percent}%` }} /></div>
|
||||
<span className="meta">{progress.phase === 'extract' ? 'extraction…' : `${progress.percent}% · ${formatMb(progress.received)} / ${formatMb(progress.total)}`}</span>
|
||||
</div>
|
||||
)}
|
||||
{progress?.phase === 'error' && <span className="meta" style={{ color: 'var(--danger)' }}>échec : {progress.message}</span>}
|
||||
</div>
|
||||
<div className="actions">
|
||||
{busy ? (
|
||||
<button className="icon-btn danger" title="Annuler" onClick={() => void cancel(model.id)}><X size={13} /></button>
|
||||
) : model.installed ? (
|
||||
<>
|
||||
{!isActive && <button className="btn small" onClick={activate}>Utiliser</button>}
|
||||
<button className="icon-btn danger" title="Supprimer du disque" onClick={() => { if (confirm(`Supprimer « ${model.name} » ?`)) void remove(model.id); }}><Trash2 size={13} /></button>
|
||||
</>
|
||||
) : (
|
||||
<button className="btn small primary" onClick={() => void download(model.id)}><Download size={13} /> Télécharger</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function ModelsSection() {
|
||||
const { models, engine, error, refresh, checkEngine } = useVoiceModels();
|
||||
useEffect(() => {
|
||||
void refresh();
|
||||
void checkEngine();
|
||||
}, [refresh, checkEngine]);
|
||||
|
||||
if (!bridge()) return <div className="card"><div className="empty">Les modèles locaux ne sont disponibles que dans l’application Electron.</div></div>;
|
||||
|
||||
const stt = models.filter((m) => m.kind === 'stt');
|
||||
const tts = models.filter((m) => m.kind === 'tts');
|
||||
return (
|
||||
<>
|
||||
<div className="card">
|
||||
<div className="section-title"><Cpu size={12} /> Moteur local (sherpa-onnx)</div>
|
||||
{engine === null ? (
|
||||
<span className="muted">vérification…</span>
|
||||
) : engine.available ? (
|
||||
<div className="test-result ok"><CheckCircle2 size={13} /> moteur prêt · version {engine.version}{engine.loaded.length ? ` · chargé : ${engine.loaded.join(', ')}` : ''}</div>
|
||||
) : (
|
||||
<div className="test-result fail"><AlertTriangle size={13} /> {engine.error}</div>
|
||||
)}
|
||||
<span className="hint" style={{ display: 'block', marginTop: 8 }}>
|
||||
Les modèles tournent sur le processeur, sans connexion, dans un processus séparé. Dossier : <code>{engine?.modelsDir}</code>
|
||||
</span>
|
||||
{error && <div className="test-result fail" style={{ marginTop: 8 }}><AlertTriangle size={13} /> {error}</div>}
|
||||
<div className="row" style={{ marginTop: 8 }}>
|
||||
<button className="btn small ghost" onClick={() => { void refresh(); void checkEngine(); }}><RefreshCw size={13} /> Actualiser</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="card">
|
||||
<div className="section-title"><Mic size={12} /> Reconnaissance vocale</div>
|
||||
<div className="list">{stt.map((m) => <ModelRow key={m.id} model={m} />)}</div>
|
||||
</div>
|
||||
<div className="card">
|
||||
<div className="section-title"><Volume2 size={12} /> Synthèse vocale</div>
|
||||
<div className="list">{tts.map((m) => <ModelRow key={m.id} model={m} />)}</div>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -10,8 +10,10 @@ import { encodeWav } from '../../services/voice/wav';
|
||||
import { useHermes } from '../../state/hermes';
|
||||
import { DEFAULT_SETTINGS, useSettings, type HudTheme } from '../../state/settings';
|
||||
import { useVoice } from '../../state/voice';
|
||||
import { installedModels, useVoiceModels } from '../../state/voiceModels';
|
||||
import { ModelsSection } from './ModelsSection';
|
||||
|
||||
type Section = 'general' | 'hermes' | 'voice' | 'speech' | 'webhook' | 'ui';
|
||||
type Section = 'general' | 'hermes' | 'voice' | 'speech' | 'models' | 'webhook' | 'ui';
|
||||
|
||||
interface Props {
|
||||
onClose: () => void;
|
||||
@@ -37,6 +39,8 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
const reset = useSettings((s) => s.reset);
|
||||
const hermesStore = useHermes();
|
||||
const micDevices = useVoice((s) => s.micDevices);
|
||||
const voiceModels = useVoiceModels((s) => s.models);
|
||||
const refreshModels = useVoiceModels((s) => s.refresh);
|
||||
const [section, setSection] = useState<Section>('hermes');
|
||||
const [hermesTest, setHermesTest] = useState<TestState>({ status: 'idle', message: '' });
|
||||
const [sttTest, setSttTest] = useState<TestState>({ status: 'idle', message: '' });
|
||||
@@ -49,8 +53,9 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
refresh();
|
||||
speechSynthesis.addEventListener?.('voiceschanged', refresh);
|
||||
void listMicrophones().then((d) => useVoice.getState().setMicDevices(d));
|
||||
void refreshModels();
|
||||
return () => speechSynthesis.removeEventListener?.('voiceschanged', refresh);
|
||||
}, []);
|
||||
}, [refreshModels]);
|
||||
|
||||
const testHermes = async () => {
|
||||
setHermesTest({ status: 'running', message: 'connexion…' });
|
||||
@@ -113,6 +118,7 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
['hermes', 'Hermes'],
|
||||
['voice', 'Micro / STT'],
|
||||
['speech', 'Voix / TTS'],
|
||||
['models', 'Modèles locaux'],
|
||||
['webhook', 'Webhook'],
|
||||
['general', 'Général'],
|
||||
['ui', 'Interface']
|
||||
@@ -195,10 +201,23 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
<div className="field">
|
||||
<label>Moteur de reconnaissance</label>
|
||||
<select className="select" value={settings.voice.provider} onChange={(e) => update({ voice: { provider: e.target.value as typeof settings.voice.provider } })}>
|
||||
<option value="local">Local dans l’application (Whisper via sherpa-onnx, hors ligne)</option>
|
||||
<option value="openai-compatible">API compatible OpenAI (Qwen3-ASR, Whisper, Speaches, LocalAI…)</option>
|
||||
<option value="browser">Reconnaissance Chromium (en ligne, secours)</option>
|
||||
</select>
|
||||
</div>
|
||||
{settings.voice.provider === 'local' && (
|
||||
<div className="field">
|
||||
<label>Modèle local</label>
|
||||
{installedModels(voiceModels, 'stt').length === 0 ? (
|
||||
<span className="hint">Aucun modèle installé. <a href="#" onClick={(e) => { e.preventDefault(); setSection('models'); }}>Téléchargez Whisper base dans « Modèles locaux »</a>.</span>
|
||||
) : (
|
||||
<select className="select" value={settings.voice.localModel} onChange={(e) => update({ voice: { localModel: e.target.value } })}>
|
||||
{installedModels(voiceModels, 'stt').map((m) => <option key={m.id} value={m.id}>{m.name}</option>)}
|
||||
</select>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{settings.voice.provider === 'openai-compatible' && (
|
||||
<>
|
||||
<div className="field">
|
||||
@@ -255,8 +274,15 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
</div>
|
||||
<Toggle on={settings.voice.handsFree} onChange={(v) => { update({ voice: { handsFree: v } }); useVoice.getState().setHandsFree(v); }} label="Mains libres au démarrage" hint="Le micro se réactive automatiquement après chaque réponse." />
|
||||
<Toggle on={settings.voice.wakeChime} onChange={(v) => update({ voice: { wakeChime: v } })} label="Signal sonore d’écoute" />
|
||||
<Toggle on={settings.voice.wakeWordEnabled} onChange={(v) => update({ voice: { wakeWordEnabled: v } })} label="Mot d’activation en mains libres" hint="Seules les phrases commençant par ce mot sont envoyées à Hermes ; le reste est ignoré. Recommandé avec la reconnaissance locale." />
|
||||
{settings.voice.wakeWordEnabled && (
|
||||
<div className="field" style={{ marginTop: 8 }}>
|
||||
<label>Mot d’activation</label>
|
||||
<input className="input" value={settings.voice.wakeWord} placeholder="jarvis" onChange={(e) => update({ voice: { wakeWord: e.target.value.toLowerCase() } })} />
|
||||
</div>
|
||||
)}
|
||||
<div className="row" style={{ marginTop: 10 }}>
|
||||
<button className="btn small" onClick={() => void testStt()} disabled={settings.voice.provider !== 'openai-compatible'}><Mic size={13} /> Tester l’API STT</button>
|
||||
<button className="btn small" onClick={() => void testStt()} disabled={settings.voice.provider === 'browser'}><Mic size={13} /> Tester la reconnaissance</button>
|
||||
</div>
|
||||
<div style={{ marginTop: 8 }}><Result t={sttTest} /></div>
|
||||
</div>
|
||||
@@ -267,6 +293,7 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
<div className="field">
|
||||
<label>Moteur de synthèse</label>
|
||||
<select className="select" value={settings.speech.provider} onChange={(e) => update({ speech: { provider: e.target.value as typeof settings.speech.provider } })}>
|
||||
<option value="local">Local dans l’application (Kokoro / Piper via sherpa-onnx, hors ligne)</option>
|
||||
<option value="openai-compatible">API compatible OpenAI /v1/audio/speech (Kokoro, Piper, OpenAI, LocalAI…)</option>
|
||||
<option value="system">Voix système Windows</option>
|
||||
<option value="google-free">Google Translate (gratuit, en ligne)</option>
|
||||
@@ -303,6 +330,33 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
{settings.speech.provider === 'local' && (
|
||||
installedModels(voiceModels, 'tts').length === 0 ? (
|
||||
<div className="field">
|
||||
<span className="hint">Aucun modèle de voix installé. <a href="#" onClick={(e) => { e.preventDefault(); setSection('models'); }}>Téléchargez Kokoro dans « Modèles locaux »</a>.</span>
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid-2">
|
||||
<div className="field">
|
||||
<label>Modèle local</label>
|
||||
<select className="select" value={settings.speech.localModel} onChange={(e) => {
|
||||
const m = voiceModels.find((x) => x.id === e.target.value);
|
||||
update({ speech: { localModel: e.target.value, localSpeaker: m?.speakers?.[0]?.id ?? 0 } });
|
||||
}}>
|
||||
{installedModels(voiceModels, 'tts').map((m) => <option key={m.id} value={m.id}>{m.name}</option>)}
|
||||
</select>
|
||||
</div>
|
||||
<div className="field">
|
||||
<label>Voix</label>
|
||||
<select className="select" value={settings.speech.localSpeaker} onChange={(e) => update({ speech: { localSpeaker: Number(e.target.value) } })}>
|
||||
{(voiceModels.find((m) => m.id === settings.speech.localModel)?.speakers ?? [{ id: 0, name: 'Voix 0', lang: '' }]).map((sp) => (
|
||||
<option key={sp.id} value={sp.id}>{sp.name}</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
)}
|
||||
{settings.speech.provider === 'system' && (
|
||||
<div className="field">
|
||||
<label>Voix système</label>
|
||||
@@ -331,6 +385,8 @@ export function SettingsDrawer({ onClose }: Props) {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{section === 'models' && <ModelsSection />}
|
||||
|
||||
{section === 'webhook' && (
|
||||
<div className="card">
|
||||
<Toggle on={settings.webhook.enabled} onChange={(v) => update({ webhook: { enabled: v } })} label="Serveur webhook local" hint="Hermes (crons, gateway, scripts) peut pousser des messages vers EveFlow." />
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { Log } from '../../lib/log';
|
||||
import { bridge } from '../../lib/bridge';
|
||||
import { httpFetch } from '../../lib/transport';
|
||||
|
||||
export type SttProvider = 'openai-compatible' | 'browser';
|
||||
export type SttProvider = 'openai-compatible' | 'browser' | 'local';
|
||||
|
||||
export interface SttConfig {
|
||||
provider: SttProvider;
|
||||
@@ -10,6 +11,8 @@ export interface SttConfig {
|
||||
model: string;
|
||||
language: string; // BCP-47, e.g. fr-FR
|
||||
prompt?: string;
|
||||
/** Catalog id of the local sherpa-onnx model (provider 'local'). */
|
||||
localModel: string;
|
||||
}
|
||||
|
||||
export function sttEndpoint(apiUrl: string): string {
|
||||
@@ -20,8 +23,20 @@ export function sttEndpoint(apiUrl: string): string {
|
||||
return `${base}/v1/audio/transcriptions`;
|
||||
}
|
||||
|
||||
/** Transcribe a WAV buffer through any OpenAI-compatible `/v1/audio/transcriptions` endpoint. */
|
||||
/** Transcribe with the in-app sherpa-onnx engine (runs in the Electron main process). */
|
||||
export async function transcribeLocal(wav: Uint8Array, config: SttConfig): Promise<string> {
|
||||
const api = bridge();
|
||||
if (!api) throw new Error('La reconnaissance locale nécessite l’application Electron.');
|
||||
if (!config.localModel) throw new Error('Aucun modèle de reconnaissance local sélectionné.');
|
||||
const started = Date.now();
|
||||
const result = await api.voice.transcribe({ modelId: config.localModel, wav, language: config.language.split('-')[0] || 'auto' });
|
||||
Log.info('stt', `local transcription in ${Date.now() - started} ms (engine ${result.durationMs} ms, audio ${result.audioSec.toFixed(1)}s)`);
|
||||
return result.text.trim();
|
||||
}
|
||||
|
||||
/** Transcribe a WAV buffer with the configured provider. */
|
||||
export async function transcribeWav(wav: Uint8Array, config: SttConfig): Promise<string> {
|
||||
if (config.provider === 'local') return transcribeLocal(wav, config);
|
||||
if (!config.apiUrl.trim()) throw new Error("URL de l'API de transcription non configurée.");
|
||||
const endpoint = sttEndpoint(config.apiUrl);
|
||||
const fields: Record<string, string> = {
|
||||
|
||||
@@ -4,11 +4,12 @@
|
||||
* and the legacy Google Translate endpoint (no key, online only).
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { bridge } from '../../lib/bridge';
|
||||
import { chunkForSpeech, cleanForSpeech, extractSentences } from '../../lib/text';
|
||||
import { httpFetch } from '../../lib/transport';
|
||||
import { audioBus } from './audioBus';
|
||||
|
||||
export type TtsProvider = 'openai-compatible' | 'system' | 'google-free' | 'off';
|
||||
export type TtsProvider = 'openai-compatible' | 'system' | 'google-free' | 'local' | 'off';
|
||||
|
||||
export interface TtsConfig {
|
||||
provider: TtsProvider;
|
||||
@@ -21,6 +22,10 @@ export interface TtsConfig {
|
||||
systemVoice: string;
|
||||
language: string;
|
||||
volume: number;
|
||||
/** Catalog id of the local sherpa-onnx voice model (provider 'local'). */
|
||||
localModel: string;
|
||||
/** Speaker id inside the local model. */
|
||||
localSpeaker: number;
|
||||
}
|
||||
|
||||
export type TtsState = 'idle' | 'loading' | 'speaking';
|
||||
@@ -159,7 +164,8 @@ export class TtsEngine {
|
||||
}
|
||||
|
||||
private prefetch(text: string, gen: number): Promise<AudioBuffer | null> {
|
||||
const task = this.config.provider === 'google-free' ? this.fetchGoogle(text) : this.fetchOpenAi(text);
|
||||
const task =
|
||||
this.config.provider === 'google-free' ? this.fetchGoogle(text) : this.config.provider === 'local' ? this.fetchLocal(text) : this.fetchOpenAi(text);
|
||||
return task
|
||||
.then(async (bytes) => {
|
||||
if (gen !== this.generation || !bytes) return null;
|
||||
@@ -199,6 +205,20 @@ export class TtsEngine {
|
||||
return res.binary;
|
||||
}
|
||||
|
||||
private async fetchLocal(text: string): Promise<Uint8Array> {
|
||||
const api = bridge();
|
||||
if (!api) throw new Error('La synthèse locale nécessite l’application Electron.');
|
||||
if (!this.config.localModel) throw new Error('Aucun modèle de voix local sélectionné.');
|
||||
const result = await api.voice.synthesize({
|
||||
modelId: this.config.localModel,
|
||||
text,
|
||||
speaker: this.config.localSpeaker,
|
||||
speed: Math.max(0.5, Math.min(2, this.config.speed || 1))
|
||||
});
|
||||
Log.debug('tts', `local synthesis ${result.durationMs} ms for ${result.audioSec.toFixed(1)}s of audio`);
|
||||
return result.wav;
|
||||
}
|
||||
|
||||
private async fetchGoogle(text: string): Promise<Uint8Array> {
|
||||
const url = `https://translate.google.com/translate_tts?ie=UTF-8&tl=${encodeURIComponent(this.config.language.split('-')[0] || 'fr')}&client=tw-ob&q=${encodeURIComponent(text.slice(0, 200))}`;
|
||||
const res = await httpFetch({
|
||||
|
||||
@@ -15,8 +15,39 @@ import type { WavResult } from './wav';
|
||||
const SENSITIVITY_RATIO: Record<number, number> = { 1: 4.5, 2: 3.4, 3: 2.6, 4: 2.0, 5: 1.6 };
|
||||
const SENSITIVITY_MIN_RMS: Record<number, number> = { 1: 0.03, 2: 0.02, 3: 0.012, 4: 0.008, 5: 0.005 };
|
||||
|
||||
/** Levenshtein distance, used to tolerate transcription slips in the wake word ("javis" for "jarvis"). */
|
||||
function editDistance(a: string, b: string): number {
|
||||
const dp = Array.from({ length: a.length + 1 }, (_, i) => [i, ...new Array<number>(b.length).fill(0)]);
|
||||
for (let j = 1; j <= b.length; j++) dp[0][j] = j;
|
||||
for (let i = 1; i <= a.length; i++) {
|
||||
for (let j = 1; j <= b.length; j++) {
|
||||
dp[i][j] = Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1, dp[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
|
||||
}
|
||||
}
|
||||
return dp[a.length][b.length];
|
||||
}
|
||||
|
||||
/** Compare the start of an utterance with the wake word, tolerant to accents, case, punctuation and small slips. */
|
||||
export function matchWakeWord(text: string, wakeWord: string): { matched: boolean; rest: string } {
|
||||
const norm = (v: string) => v.normalize('NFD').replace(/[\u0300-\u036f]/g, '').toLowerCase();
|
||||
const target = norm(wakeWord).split(/[\s\p{P}]+/u).filter(Boolean);
|
||||
if (target.length === 0) return { matched: true, rest: text.trim() };
|
||||
const original = text.trim().replace(/^[\s\p{P}]+/u, '');
|
||||
const words = original.split(/\s+/);
|
||||
const head = words.slice(0, target.length).map((w) => norm(w).replace(/[\p{P}]+/gu, ''));
|
||||
if (head.length < target.length) return { matched: false, rest: '' };
|
||||
const joinedHead = head.join(' ');
|
||||
const joinedTarget = target.join(' ');
|
||||
const tolerance = joinedTarget.length >= 6 ? 2 : joinedTarget.length >= 4 ? 1 : 0;
|
||||
if (editDistance(joinedHead, joinedTarget) > tolerance) return { matched: false, rest: '' };
|
||||
const rest = words.slice(target.length).join(' ').replace(/^[\s\p{P}]+/u, '');
|
||||
return { matched: true, rest };
|
||||
}
|
||||
|
||||
class VoiceController {
|
||||
private capture = new MicCapture();
|
||||
/** After a bare wake word, the next utterance is accepted without the wake word. */
|
||||
private attentionUntil = 0;
|
||||
private browser = new BrowserRecognizer();
|
||||
private handsFreeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private stopping = false;
|
||||
@@ -169,10 +200,28 @@ class VoiceController {
|
||||
this.playChime(false);
|
||||
const settings = useSettings.getState().settings.voice;
|
||||
try {
|
||||
const text = await transcribeWav(wav.bytes, settings);
|
||||
let text = await transcribeWav(wav.bytes, settings);
|
||||
voice.setTranscript(text);
|
||||
voice.setPhase('off');
|
||||
Log.info('voice', `transcript (${wav.durationSec.toFixed(1)}s): ${text}`);
|
||||
if (voice.handsFree && settings.wakeWordEnabled && Date.now() > this.attentionUntil) {
|
||||
const { matched, rest } = matchWakeWord(text, settings.wakeWord || 'jarvis');
|
||||
if (!matched) {
|
||||
Log.debug('voice', 'utterance ignored: no wake word');
|
||||
voice.setTranscript('');
|
||||
useChat.getState().setHud('idle');
|
||||
this.scheduleHandsFree(200);
|
||||
return;
|
||||
}
|
||||
if (!rest.trim()) {
|
||||
this.attentionUntil = Date.now() + 10_000;
|
||||
useChat.getState().setHud('listening');
|
||||
speech.say('Oui ?');
|
||||
return;
|
||||
}
|
||||
text = rest;
|
||||
}
|
||||
this.attentionUntil = 0;
|
||||
if (text.trim()) {
|
||||
await sendMessage(text, [], 'voice');
|
||||
} else {
|
||||
|
||||
@@ -16,6 +16,9 @@ export interface VoiceSettings extends SttConfig {
|
||||
silenceMs: number;
|
||||
sensitivity: number; // 1 (low) .. 5 (high)
|
||||
wakeChime: boolean;
|
||||
/** Hands-free: only react to utterances starting with this word (local STT recommended). */
|
||||
wakeWordEnabled: boolean;
|
||||
wakeWord: string;
|
||||
}
|
||||
|
||||
export interface SpeechSettings extends TtsConfig {
|
||||
@@ -77,7 +80,10 @@ export const DEFAULT_SETTINGS: Settings = {
|
||||
micDeviceId: '',
|
||||
silenceMs: 900,
|
||||
sensitivity: 3,
|
||||
wakeChime: true
|
||||
wakeChime: true,
|
||||
wakeWordEnabled: false,
|
||||
wakeWord: 'jarvis',
|
||||
localModel: 'whisper-base'
|
||||
},
|
||||
speech: {
|
||||
provider: 'openai-compatible',
|
||||
@@ -90,6 +96,8 @@ export const DEFAULT_SETTINGS: Settings = {
|
||||
systemVoice: '',
|
||||
language: 'fr-FR',
|
||||
volume: 1,
|
||||
localModel: 'kokoro-v1',
|
||||
localSpeaker: 30,
|
||||
autoSpeak: true,
|
||||
speakIncoming: true
|
||||
},
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
import { create } from 'zustand';
|
||||
import type { VoiceDownloadProgress, VoiceEngineStatus, VoiceModelStatus } from '../../shared/voice';
|
||||
import { bridge } from '../lib/bridge';
|
||||
import { Log } from '../lib/log';
|
||||
|
||||
interface VoiceModelsStore {
|
||||
models: VoiceModelStatus[];
|
||||
progress: Record<string, VoiceDownloadProgress>;
|
||||
engine: VoiceEngineStatus | null;
|
||||
error: string | null;
|
||||
refresh: () => Promise<void>;
|
||||
checkEngine: () => Promise<void>;
|
||||
download: (id: string) => Promise<void>;
|
||||
cancel: (id: string) => Promise<void>;
|
||||
remove: (id: string) => Promise<void>;
|
||||
subscribe: () => () => void;
|
||||
}
|
||||
|
||||
export const useVoiceModels = create<VoiceModelsStore>((set, get) => ({
|
||||
models: [],
|
||||
progress: {},
|
||||
engine: null,
|
||||
error: null,
|
||||
|
||||
refresh: async () => {
|
||||
const api = bridge();
|
||||
if (!api) return;
|
||||
try {
|
||||
set({ models: await api.voice.listModels(), error: null });
|
||||
} catch (err) {
|
||||
set({ error: (err as Error).message });
|
||||
}
|
||||
},
|
||||
checkEngine: async () => {
|
||||
const api = bridge();
|
||||
if (!api) return;
|
||||
try {
|
||||
set({ engine: await api.voice.status() });
|
||||
} catch (err) {
|
||||
set({ engine: { available: false, error: (err as Error).message, loaded: [], modelsDir: '' } });
|
||||
}
|
||||
},
|
||||
download: async (id) => {
|
||||
const api = bridge();
|
||||
if (!api) return;
|
||||
set((s) => ({ progress: { ...s.progress, [id]: { id, phase: 'download', received: 0, total: 0, percent: 0 } }, error: null }));
|
||||
try {
|
||||
await api.voice.downloadModel(id);
|
||||
} catch (err) {
|
||||
const message = (err as Error).message;
|
||||
Log.error('voice', `download ${id} failed: ${message}`);
|
||||
set({ error: message });
|
||||
} finally {
|
||||
await get().refresh();
|
||||
}
|
||||
},
|
||||
cancel: async (id) => {
|
||||
await bridge()?.voice.cancelDownload(id);
|
||||
},
|
||||
remove: async (id) => {
|
||||
const api = bridge();
|
||||
if (!api) return;
|
||||
try {
|
||||
set({ models: await api.voice.removeModel(id) });
|
||||
} catch (err) {
|
||||
set({ error: (err as Error).message });
|
||||
}
|
||||
},
|
||||
subscribe: () => {
|
||||
const api = bridge();
|
||||
if (!api) return () => undefined;
|
||||
return api.voice.onProgress((p) => {
|
||||
set((s) => {
|
||||
const progress = { ...s.progress, [p.id]: p };
|
||||
if (p.phase === 'done' || p.phase === 'error' || p.phase === 'cancelled') {
|
||||
setTimeout(() => {
|
||||
set((s2) => {
|
||||
const next = { ...s2.progress };
|
||||
delete next[p.id];
|
||||
return { progress: next };
|
||||
});
|
||||
void get().refresh();
|
||||
}, 1500);
|
||||
}
|
||||
return { progress };
|
||||
});
|
||||
});
|
||||
}
|
||||
}));
|
||||
|
||||
export const installedModels = (models: VoiceModelStatus[], kind: 'stt' | 'tts') => models.filter((m) => m.kind === kind && m.installed);
|
||||
Reference in new issue
Block a user