mirror of
https://github.com/R0m1k3/EveFlow.git
synced 2026-10-20 05:38:14 +02:00
feat: local offline voice models (sherpa-onnx) with model manager and wake word (v2.1.0)
- electron/voice: model catalog (Whisper base/small/turbo, SenseVoice, Kokoro v1.0, Piper fr), streaming tar.bz2 downloader with progress, sherpa-onnx worker running in an Electron utilityProcess (transcribe / synthesize / status / unload), IPC + bridge. - Renderer: 'local' providers for STT and TTS, Settings → Modèles locaux (download, progress, delete, activate), speaker selection, hands-free wake word with a tolerant matcher (accents, punctuation, edit distance) and attention window after a bare wake word. - Packaging: native addon and worker unpacked from the asar; release 2.1.0. Validated on Linux: Kokoro (fr) → Whisper base round trip, Piper (fr) → Whisper round trip. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y
This commit is contained in:
22 files changed
+1361
-16
No files matched your search
@@ -8,6 +8,8 @@ import { registerTelemetryIpc } from './ipc/telemetry';
|
||||
import { getSharedDirectory, registerFilesIpc } from './ipc/files';
|
||||
import { createMainWindow, getMainWindow, getWindowMode, setWindowMode, toggleWindowVisibility } from './window';
|
||||
import { getWebhookStatus, startWebhookServer, stopWebhookServer } from './webhook';
|
||||
import { registerVoiceIpc } from './voice/ipc';
|
||||
import { stopEngine } from './voice/engine';
|
||||
|
||||
// Audio playback must never be blocked behind a user gesture (TTS starts on incoming events).
|
||||
app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required');
|
||||
@@ -149,6 +151,7 @@ app.whenReady().then(async () => {
|
||||
registerTelemetryIpc();
|
||||
registerFilesIpc();
|
||||
registerCoreIpc();
|
||||
registerVoiceIpc();
|
||||
createMainWindow();
|
||||
createTray();
|
||||
registerShortcuts();
|
||||
@@ -162,6 +165,7 @@ app.whenReady().then(async () => {
|
||||
app.on('will-quit', () => {
|
||||
globalShortcut.unregisterAll();
|
||||
abortAllStreams();
|
||||
stopEngine();
|
||||
void stopWebhookServer();
|
||||
});
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ import {
|
||||
type WindowMode
|
||||
} from '../shared/ipc';
|
||||
import type { EveFlowBridge, Unsubscribe } from '../shared/bridge';
|
||||
import { VOICE_IPC, type SynthesizeRequest, type SynthesizeResult, type TranscribeRequest, type TranscribeResult, type VoiceDownloadProgress, type VoiceEngineStatus, type VoiceModelStatus } from '../shared/voice';
|
||||
|
||||
function subscribe<T>(channel: string, callback: (payload: T) => void): Unsubscribe {
|
||||
const listener = (_event: Electron.IpcRendererEvent, payload: T) => callback(payload);
|
||||
@@ -57,6 +58,17 @@ const api: EveFlowBridge = {
|
||||
},
|
||||
hotkeys: {
|
||||
on: (cb: (event: HotkeyEvent) => void) => subscribe<HotkeyEvent>(IPC.hotkey, cb)
|
||||
},
|
||||
voice: {
|
||||
status: () => ipcRenderer.invoke(VOICE_IPC.status) as Promise<VoiceEngineStatus>,
|
||||
listModels: () => ipcRenderer.invoke(VOICE_IPC.modelsList) as Promise<VoiceModelStatus[]>,
|
||||
downloadModel: (id: string) => ipcRenderer.invoke(VOICE_IPC.modelsDownload, id) as Promise<VoiceModelStatus>,
|
||||
cancelDownload: (id: string) => ipcRenderer.invoke(VOICE_IPC.modelsCancel, id) as Promise<boolean>,
|
||||
removeModel: (id: string) => ipcRenderer.invoke(VOICE_IPC.modelsRemove, id) as Promise<VoiceModelStatus[]>,
|
||||
onProgress: (cb: (progress: VoiceDownloadProgress) => void) => subscribe<VoiceDownloadProgress>(VOICE_IPC.modelsProgress, cb),
|
||||
transcribe: (req: TranscribeRequest) => ipcRenderer.invoke(VOICE_IPC.transcribe, req) as Promise<TranscribeResult>,
|
||||
synthesize: (req: SynthesizeRequest) => ipcRenderer.invoke(VOICE_IPC.synthesize, req) as Promise<SynthesizeResult>,
|
||||
unload: (id?: string) => ipcRenderer.invoke(VOICE_IPC.unload, id) as Promise<unknown>
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
import type { VoiceModelSpec, VoiceSpeaker } from '../../shared/voice';
|
||||
|
||||
const ASR = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models';
|
||||
const TTS = 'https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models';
|
||||
|
||||
const KOKORO_SPEAKERS: VoiceSpeaker[] = [
|
||||
{ id: 30, name: 'Siwis (femme, français)', lang: 'fr' },
|
||||
{ id: 3, name: 'Heart (femme, anglais US)', lang: 'en' },
|
||||
{ id: 2, name: 'Bella (femme, anglais US)', lang: 'en' },
|
||||
{ id: 7, name: 'Nova (femme, anglais US)', lang: 'en' },
|
||||
{ id: 11, name: 'Adam (homme, anglais US)', lang: 'en' },
|
||||
{ id: 16, name: 'Michael (homme, anglais US)', lang: 'en' },
|
||||
{ id: 17, name: 'Onyx (homme, anglais US)', lang: 'en' },
|
||||
{ id: 21, name: 'Emma (femme, anglais UK)', lang: 'en' },
|
||||
{ id: 24, name: 'Daniel (homme, anglais UK)', lang: 'en' },
|
||||
{ id: 26, name: 'George (homme, anglais UK)', lang: 'en' }
|
||||
];
|
||||
|
||||
export const VOICE_CATALOG: VoiceModelSpec[] = [
|
||||
{
|
||||
id: 'whisper-base',
|
||||
kind: 'stt',
|
||||
engine: 'whisper',
|
||||
name: 'Whisper base (multilingue)',
|
||||
description: 'Rapide sur CPU, bonne qualité en français pour des commandes courtes.',
|
||||
languages: ['fr', 'en', 'multi'],
|
||||
sizeMb: 208,
|
||||
url: `${ASR}/sherpa-onnx-whisper-base.tar.bz2`,
|
||||
dir: 'sherpa-onnx-whisper-base',
|
||||
files: ['base-encoder.int8.onnx', 'base-decoder.int8.onnx', 'base-tokens.txt'],
|
||||
recommended: true
|
||||
},
|
||||
{
|
||||
id: 'whisper-small',
|
||||
kind: 'stt',
|
||||
engine: 'whisper',
|
||||
name: 'Whisper small (multilingue)',
|
||||
description: 'Plus précis que base, environ trois fois plus lent.',
|
||||
languages: ['fr', 'en', 'multi'],
|
||||
sizeMb: 640,
|
||||
url: `${ASR}/sherpa-onnx-whisper-small.tar.bz2`,
|
||||
dir: 'sherpa-onnx-whisper-small',
|
||||
files: ['small-encoder.int8.onnx', 'small-decoder.int8.onnx', 'small-tokens.txt']
|
||||
},
|
||||
{
|
||||
id: 'whisper-turbo',
|
||||
kind: 'stt',
|
||||
engine: 'whisper',
|
||||
name: 'Whisper large-v3 turbo (multilingue)',
|
||||
description: 'La meilleure précision ; demande un processeur puissant.',
|
||||
languages: ['fr', 'en', 'multi'],
|
||||
sizeMb: 564,
|
||||
url: `${ASR}/sherpa-onnx-whisper-turbo.tar.bz2`,
|
||||
dir: 'sherpa-onnx-whisper-turbo',
|
||||
files: ['turbo-encoder.int8.onnx', 'turbo-decoder.int8.onnx', 'turbo-tokens.txt']
|
||||
},
|
||||
{
|
||||
id: 'sense-voice',
|
||||
kind: 'stt',
|
||||
engine: 'sense-voice',
|
||||
name: 'SenseVoice small (zh / en / ja / ko)',
|
||||
description: 'Très rapide, mais ne comprend pas le français.',
|
||||
languages: ['en', 'zh', 'ja', 'ko'],
|
||||
sizeMb: 163,
|
||||
url: `${ASR}/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2`,
|
||||
dir: 'sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17',
|
||||
files: ['model.int8.onnx', 'tokens.txt']
|
||||
},
|
||||
{
|
||||
id: 'kokoro-v1',
|
||||
kind: 'tts',
|
||||
engine: 'kokoro',
|
||||
name: 'Kokoro v1.0 multilingue',
|
||||
description: 'Voix très naturelle, une voix française (Siwis) et de nombreuses voix anglaises. 24 kHz.',
|
||||
languages: ['fr', 'en', 'multi'],
|
||||
sizeMb: 349,
|
||||
url: `${TTS}/kokoro-multi-lang-v1_0.tar.bz2`,
|
||||
dir: 'kokoro-multi-lang-v1_0',
|
||||
files: ['model.onnx', 'voices.bin', 'tokens.txt', 'lexicon-us-en.txt', 'lexicon-zh.txt', 'espeak-ng-data/phontab'],
|
||||
speakers: KOKORO_SPEAKERS,
|
||||
sampleRate: 24000,
|
||||
recommended: true
|
||||
},
|
||||
{
|
||||
id: 'piper-fr-siwis',
|
||||
kind: 'tts',
|
||||
engine: 'piper',
|
||||
name: 'Piper Siwis (femme, français)',
|
||||
description: 'Léger et instantané, rendu un peu plus mécanique que Kokoro. 22 kHz.',
|
||||
languages: ['fr'],
|
||||
sizeMb: 67,
|
||||
url: `${TTS}/vits-piper-fr_FR-siwis-medium.tar.bz2`,
|
||||
dir: 'vits-piper-fr_FR-siwis-medium',
|
||||
files: ['fr_FR-siwis-medium.onnx', 'tokens.txt', 'espeak-ng-data/phontab'],
|
||||
speakers: [{ id: 0, name: 'Siwis', lang: 'fr' }],
|
||||
sampleRate: 22050
|
||||
},
|
||||
{
|
||||
id: 'piper-fr-tom',
|
||||
kind: 'tts',
|
||||
engine: 'piper',
|
||||
name: 'Piper Tom (homme, français)',
|
||||
description: 'Voix masculine légère et instantanée. 22 kHz.',
|
||||
languages: ['fr'],
|
||||
sizeMb: 67,
|
||||
url: `${TTS}/vits-piper-fr_FR-tom-medium.tar.bz2`,
|
||||
dir: 'vits-piper-fr_FR-tom-medium',
|
||||
files: ['fr_FR-tom-medium.onnx', 'tokens.txt', 'espeak-ng-data/phontab'],
|
||||
speakers: [{ id: 0, name: 'Tom', lang: 'fr' }],
|
||||
sampleRate: 22050
|
||||
},
|
||||
{
|
||||
id: 'piper-fr-upmc',
|
||||
kind: 'tts',
|
||||
engine: 'piper',
|
||||
name: 'Piper UPMC (Jessica et Pierre, français)',
|
||||
description: 'Deux voix françaises, une féminine et une masculine. 22 kHz.',
|
||||
languages: ['fr'],
|
||||
sizeMb: 80,
|
||||
url: `${TTS}/vits-piper-fr_FR-upmc-medium.tar.bz2`,
|
||||
dir: 'vits-piper-fr_FR-upmc-medium',
|
||||
files: ['fr_FR-upmc-medium.onnx', 'tokens.txt', 'espeak-ng-data/phontab'],
|
||||
speakers: [
|
||||
{ id: 0, name: 'Jessica (femme)', lang: 'fr' },
|
||||
{ id: 1, name: 'Pierre (homme)', lang: 'fr' }
|
||||
],
|
||||
sampleRate: 22050
|
||||
}
|
||||
];
|
||||
|
||||
export function findModel(id: string): VoiceModelSpec | undefined {
|
||||
return VOICE_CATALOG.find((m) => m.id === id);
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Host side of the voice worker: spawns the utility process on demand, correlates
|
||||
* requests and responses, restarts the worker if it crashes.
|
||||
*/
|
||||
import { utilityProcess, type UtilityProcess } from 'electron';
|
||||
import path from 'node:path';
|
||||
import type { SynthesizeRequest, SynthesizeResult, TranscribeRequest, TranscribeResult, VoiceEngineStatus } from '../../shared/voice';
|
||||
import { findModel } from './catalog';
|
||||
import { isInstalled, modelDir, modelsDir } from './models';
|
||||
import { log } from '../logger';
|
||||
|
||||
interface Pending {
|
||||
resolve: (value: unknown) => void;
|
||||
reject: (err: Error) => void;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
let child: UtilityProcess | null = null;
|
||||
let nextId = 1;
|
||||
const pending = new Map<number, Pending>();
|
||||
|
||||
function spawn(): UtilityProcess {
|
||||
if (child) return child;
|
||||
const script = path.join(__dirname, 'voice-worker.js');
|
||||
child = utilityProcess.fork(script, [], { serviceName: 'eveflow-voice', stdio: 'pipe' });
|
||||
child.stdout?.on('data', (d: Buffer) => log('DEBUG', 'voice-worker', d.toString().trim()));
|
||||
child.stderr?.on('data', (d: Buffer) => log('WARN', 'voice-worker', d.toString().trim()));
|
||||
child.on('message', (msg: { id: number; ok: boolean; result?: unknown; error?: string }) => {
|
||||
const p = pending.get(msg.id);
|
||||
if (!p) return;
|
||||
pending.delete(msg.id);
|
||||
clearTimeout(p.timer);
|
||||
if (msg.ok) p.resolve(msg.result);
|
||||
else p.reject(new Error(msg.error ?? 'erreur moteur vocal'));
|
||||
});
|
||||
child.on('exit', (code) => {
|
||||
log(code === 0 ? 'INFO' : 'ERROR', 'voice-worker', `exited with code ${code}`);
|
||||
child = null;
|
||||
for (const [id, p] of pending) {
|
||||
clearTimeout(p.timer);
|
||||
p.reject(new Error('Le moteur vocal local s’est arrêté de façon inattendue.'));
|
||||
pending.delete(id);
|
||||
}
|
||||
});
|
||||
log('INFO', 'voice', 'voice worker started');
|
||||
return child;
|
||||
}
|
||||
|
||||
function request<T>(message: Record<string, unknown>, timeoutMs = 120_000): Promise<T> {
|
||||
const proc = spawn();
|
||||
const id = nextId++;
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
pending.delete(id);
|
||||
reject(new Error('Délai dépassé pour le moteur vocal local.'));
|
||||
}, timeoutMs);
|
||||
pending.set(id, { resolve: (v) => resolve(v as T), reject, timer });
|
||||
proc.postMessage({ id, ...message });
|
||||
});
|
||||
}
|
||||
|
||||
function modelRef(modelId: string) {
|
||||
const spec = findModel(modelId);
|
||||
if (!spec) throw new Error(`Modèle inconnu : ${modelId}`);
|
||||
if (!isInstalled(spec)) throw new Error(`Modèle « ${spec.name} » non installé. Téléchargez-le dans Paramètres → Modèles.`);
|
||||
return { id: spec.id, engine: spec.engine, dir: modelDir(spec), files: spec.files };
|
||||
}
|
||||
|
||||
export async function engineStatus(): Promise<VoiceEngineStatus> {
|
||||
try {
|
||||
const status = await request<Omit<VoiceEngineStatus, 'modelsDir'>>({ type: 'status' }, 20_000);
|
||||
return { ...status, modelsDir: modelsDir() };
|
||||
} catch (err) {
|
||||
return { available: false, error: (err as Error).message, loaded: [], modelsDir: modelsDir() };
|
||||
}
|
||||
}
|
||||
|
||||
export function transcribe(req: TranscribeRequest): Promise<TranscribeResult> {
|
||||
return request<TranscribeResult>({ type: 'transcribe', model: modelRef(req.modelId), wav: req.wav, language: req.language }, 180_000);
|
||||
}
|
||||
|
||||
export function synthesize(req: SynthesizeRequest): Promise<SynthesizeResult> {
|
||||
return request<SynthesizeResult>({ type: 'synthesize', model: modelRef(req.modelId), text: req.text, speaker: req.speaker, speed: req.speed }, 180_000);
|
||||
}
|
||||
|
||||
export function unload(modelId?: string): Promise<unknown> {
|
||||
if (!child) return Promise.resolve({ ok: true });
|
||||
return request({ type: 'unload', modelId }, 20_000);
|
||||
}
|
||||
|
||||
export function stopEngine(): void {
|
||||
if (child) {
|
||||
child.kill();
|
||||
child = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { ipcMain } from 'electron';
|
||||
import { VOICE_IPC, type SynthesizeRequest, type TranscribeRequest } from '../../shared/voice';
|
||||
import { engineStatus, synthesize, transcribe, unload } from './engine';
|
||||
import { cancelDownload, downloadModel, listModels, removeModel } from './models';
|
||||
|
||||
export function registerVoiceIpc(): void {
|
||||
ipcMain.handle(VOICE_IPC.status, () => engineStatus());
|
||||
ipcMain.handle(VOICE_IPC.modelsList, () => listModels());
|
||||
ipcMain.handle(VOICE_IPC.modelsDownload, (event, id: string) => downloadModel(id, event.sender));
|
||||
ipcMain.handle(VOICE_IPC.modelsCancel, (_e, id: string) => {
|
||||
cancelDownload(id);
|
||||
return true;
|
||||
});
|
||||
ipcMain.handle(VOICE_IPC.modelsRemove, async (_e, id: string) => {
|
||||
await unload(id).catch(() => undefined);
|
||||
await removeModel(id);
|
||||
return listModels();
|
||||
});
|
||||
ipcMain.handle(VOICE_IPC.transcribe, (_e, req: TranscribeRequest) => transcribe(req));
|
||||
ipcMain.handle(VOICE_IPC.synthesize, (_e, req: SynthesizeRequest) => synthesize(req));
|
||||
ipcMain.handle(VOICE_IPC.unload, (_e, id?: string) => unload(id));
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
/**
|
||||
* Model manager: downloads sherpa-onnx model archives (tar.bz2) into userData/models,
|
||||
* extracts them, verifies required files and reports progress to the renderer.
|
||||
*/
|
||||
import { app, type WebContents } from 'electron';
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { pipeline } from 'node:stream/promises';
|
||||
import { Readable, Transform } from 'node:stream';
|
||||
import * as tar from 'tar';
|
||||
import unbzip2 from 'unbzip2-stream';
|
||||
import { VOICE_IPC, type VoiceDownloadProgress, type VoiceModelSpec, type VoiceModelStatus } from '../../shared/voice';
|
||||
import { VOICE_CATALOG, findModel } from './catalog';
|
||||
import { log } from '../logger';
|
||||
|
||||
const active = new Map<string, AbortController>();
|
||||
|
||||
export function modelsDir(): string {
|
||||
const dir = path.join(app.getPath('userData'), 'models');
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
export function modelDir(spec: VoiceModelSpec): string {
|
||||
return path.join(modelsDir(), spec.dir);
|
||||
}
|
||||
|
||||
export function isInstalled(spec: VoiceModelSpec): boolean {
|
||||
const dir = modelDir(spec);
|
||||
return spec.files.every((f) => fs.existsSync(path.join(dir, f)));
|
||||
}
|
||||
|
||||
function dirSize(dir: string): number {
|
||||
let total = 0;
|
||||
try {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) total += dirSize(full);
|
||||
else total += fs.statSync(full).size;
|
||||
}
|
||||
} catch {
|
||||
/* missing */
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
export function listModels(): VoiceModelStatus[] {
|
||||
return VOICE_CATALOG.map((spec) => {
|
||||
const installed = isInstalled(spec);
|
||||
return {
|
||||
...spec,
|
||||
installed,
|
||||
downloading: active.has(spec.id),
|
||||
installedBytes: installed ? dirSize(modelDir(spec)) : 0
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function removeModel(id: string): Promise<void> {
|
||||
const spec = findModel(id);
|
||||
if (!spec) throw new Error(`Modèle inconnu : ${id}`);
|
||||
await fs.promises.rm(modelDir(spec), { recursive: true, force: true });
|
||||
log('INFO', 'voice', `model removed: ${id}`);
|
||||
}
|
||||
|
||||
export function cancelDownload(id: string): void {
|
||||
active.get(id)?.abort();
|
||||
}
|
||||
|
||||
export async function downloadModel(id: string, sender: WebContents | null): Promise<VoiceModelStatus> {
|
||||
const spec = findModel(id);
|
||||
if (!spec) throw new Error(`Modèle inconnu : ${id}`);
|
||||
if (active.has(id)) throw new Error('Téléchargement déjà en cours');
|
||||
const controller = new AbortController();
|
||||
active.set(id, controller);
|
||||
const report = (progress: Omit<VoiceDownloadProgress, 'id'>) => {
|
||||
if (sender && !sender.isDestroyed()) sender.send(VOICE_IPC.modelsProgress, { id, ...progress });
|
||||
};
|
||||
|
||||
const tmpDir = path.join(modelsDir(), `.tmp-${spec.id}`);
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true });
|
||||
await fs.promises.mkdir(tmpDir, { recursive: true });
|
||||
|
||||
try {
|
||||
log('INFO', 'voice', `downloading ${spec.id} from ${spec.url}`);
|
||||
const response = await fetch(spec.url, { signal: controller.signal, redirect: 'follow' });
|
||||
if (!response.ok || !response.body) throw new Error(`HTTP ${response.status} lors du téléchargement`);
|
||||
const total = Number(response.headers.get('content-length')) || Math.round(spec.sizeMb * 1024 * 1024);
|
||||
let received = 0;
|
||||
let lastReport = 0;
|
||||
const counter = new Transform({
|
||||
transform(chunk: Buffer, _enc, cb) {
|
||||
received += chunk.length;
|
||||
const now = Date.now();
|
||||
if (now - lastReport > 250) {
|
||||
lastReport = now;
|
||||
report({ phase: 'download', received, total, percent: Math.min(99, Math.round((received / total) * 100)) });
|
||||
}
|
||||
cb(null, chunk);
|
||||
}
|
||||
});
|
||||
|
||||
const source = Readable.fromWeb(response.body as import('node:stream/web').ReadableStream<Uint8Array>);
|
||||
if (spec.url.endsWith('.tar.bz2')) {
|
||||
report({ phase: 'download', received: 0, total, percent: 0 });
|
||||
await pipeline(source, counter, unbzip2(), tar.x({ cwd: tmpDir }), { signal: controller.signal });
|
||||
} else {
|
||||
const target = path.join(tmpDir, spec.dir, path.basename(spec.url));
|
||||
await fs.promises.mkdir(path.dirname(target), { recursive: true });
|
||||
await pipeline(source, counter, fs.createWriteStream(target), { signal: controller.signal });
|
||||
}
|
||||
report({ phase: 'extract', received: total, total, percent: 99 });
|
||||
|
||||
const extracted = path.join(tmpDir, spec.dir);
|
||||
const missing = spec.files.filter((f) => !fs.existsSync(path.join(extracted, f)));
|
||||
if (missing.length) throw new Error(`Archive incomplète, fichiers manquants : ${missing.join(', ')}`);
|
||||
|
||||
const finalDir = modelDir(spec);
|
||||
await fs.promises.rm(finalDir, { recursive: true, force: true });
|
||||
await fs.promises.rename(extracted, finalDir);
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true });
|
||||
report({ phase: 'done', received: total, total, percent: 100 });
|
||||
log('INFO', 'voice', `model installed: ${spec.id}`);
|
||||
} catch (err) {
|
||||
await fs.promises.rm(tmpDir, { recursive: true, force: true }).catch(() => undefined);
|
||||
const e = err as Error;
|
||||
const cancelled = e.name === 'AbortError' || controller.signal.aborted;
|
||||
report({ phase: cancelled ? 'cancelled' : 'error', received: 0, total: 0, percent: 0, message: cancelled ? 'annulé' : e.message });
|
||||
log(cancelled ? 'INFO' : 'ERROR', 'voice', `download ${spec.id} ${cancelled ? 'cancelled' : 'failed: ' + e.message}`);
|
||||
if (!cancelled) throw new Error(e.message);
|
||||
} finally {
|
||||
active.delete(id);
|
||||
}
|
||||
return listModels().find((m) => m.id === id)!;
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
/**
|
||||
* Voice worker: runs sherpa-onnx (speech recognition and synthesis) in an Electron utility
|
||||
* process so heavy inference never blocks the main process. Also runnable with
|
||||
* `child_process.fork` (advanced serialization) for local tests.
|
||||
*/
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import type { VoiceEngineKind } from '../../shared/voice';
|
||||
|
||||
interface ModelRef {
|
||||
id: string;
|
||||
engine: VoiceEngineKind;
|
||||
dir: string;
|
||||
files: string[];
|
||||
}
|
||||
|
||||
type Request =
|
||||
| { id: number; type: 'status' }
|
||||
| { id: number; type: 'transcribe'; model: ModelRef; wav: Uint8Array; language: string }
|
||||
| { id: number; type: 'synthesize'; model: ModelRef; text: string; speaker: number; speed: number }
|
||||
| { id: number; type: 'unload'; modelId?: string };
|
||||
|
||||
type Response = { id: number; ok: true; result: unknown } | { id: number; ok: false; error: string };
|
||||
|
||||
// ── sherpa-onnx loading (lazy, so a missing native package is reported, not fatal) ──
|
||||
type Sherpa = {
|
||||
OfflineRecognizer: new (config: unknown) => {
|
||||
createStream: () => { acceptWaveform: (w: { sampleRate: number; samples: Float32Array }) => void };
|
||||
decode: (s: unknown) => void;
|
||||
getResult: (s: unknown) => { text: string; lang?: string };
|
||||
};
|
||||
OfflineTts: new (config: unknown) => {
|
||||
numSpeakers: number;
|
||||
sampleRate: number;
|
||||
generate: (req: { text: string; sid: number; speed: number }) => { samples: Float32Array; sampleRate: number };
|
||||
};
|
||||
version: string;
|
||||
};
|
||||
|
||||
let sherpa: Sherpa | null = null;
|
||||
let loadError: string | null = null;
|
||||
|
||||
function loadSherpa(): Sherpa {
|
||||
if (sherpa) return sherpa;
|
||||
if (loadError) throw new Error(loadError);
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-require-imports
|
||||
sherpa = require('sherpa-onnx-node') as Sherpa;
|
||||
return sherpa;
|
||||
} catch (err) {
|
||||
loadError = `Module natif sherpa-onnx indisponible : ${(err as Error).message}`;
|
||||
throw new Error(loadError);
|
||||
}
|
||||
}
|
||||
|
||||
const threads = Math.max(2, Math.min(6, Math.floor(os.cpus().length / 2)));
|
||||
|
||||
// ── caches ────────────────────────────────────────────────────────────────
|
||||
const recognizers = new Map<string, InstanceType<Sherpa['OfflineRecognizer']>>();
|
||||
const synthesizers = new Map<string, InstanceType<Sherpa['OfflineTts']>>();
|
||||
|
||||
function whisperPrefix(model: ModelRef): string {
|
||||
const encoder = model.files.find((f) => f.includes('-encoder'));
|
||||
return encoder ? encoder.slice(0, encoder.indexOf('-encoder')) : 'base';
|
||||
}
|
||||
|
||||
function getRecognizer(model: ModelRef, language: string) {
|
||||
const lang = language === 'auto' ? '' : language.split('-')[0].toLowerCase();
|
||||
const key = `${model.id}:${lang}`;
|
||||
const cached = recognizers.get(key);
|
||||
if (cached) return cached;
|
||||
const s = loadSherpa();
|
||||
const p = (f: string) => path.join(model.dir, f);
|
||||
let modelConfig: Record<string, unknown>;
|
||||
switch (model.engine) {
|
||||
case 'whisper': {
|
||||
const prefix = whisperPrefix(model);
|
||||
modelConfig = {
|
||||
whisper: { encoder: p(`${prefix}-encoder.int8.onnx`), decoder: p(`${prefix}-decoder.int8.onnx`), language: lang, task: 'transcribe', tailPaddings: -1 },
|
||||
tokens: p(`${prefix}-tokens.txt`)
|
||||
};
|
||||
break;
|
||||
}
|
||||
case 'sense-voice':
|
||||
modelConfig = { senseVoice: { model: p('model.int8.onnx'), language: lang || 'auto', useInverseTextNormalization: 1 }, tokens: p('tokens.txt') };
|
||||
break;
|
||||
case 'nemo-transducer':
|
||||
modelConfig = {
|
||||
transducer: { encoder: p('encoder.int8.onnx'), decoder: p('decoder.int8.onnx'), joiner: p('joiner.int8.onnx') },
|
||||
tokens: p('tokens.txt'),
|
||||
modelType: 'nemo_transducer'
|
||||
};
|
||||
break;
|
||||
default:
|
||||
throw new Error(`Moteur STT non supporté : ${model.engine}`);
|
||||
}
|
||||
for (const [k, v] of Object.entries({ numThreads: threads, provider: 'cpu', debug: 0 })) modelConfig[k] = v;
|
||||
// Whisper keeps one recognizer per language; other engines ignore the language key.
|
||||
for (const [k, r] of recognizers) if (k.startsWith(`${model.id}:`)) recognizers.delete(k) && void r;
|
||||
const recognizer = new s.OfflineRecognizer({ featConfig: { sampleRate: 16000, featureDim: 80 }, modelConfig, decodingMethod: 'greedy_search' });
|
||||
recognizers.set(key, recognizer);
|
||||
return recognizer;
|
||||
}
|
||||
|
||||
function getSynthesizer(model: ModelRef) {
|
||||
const cached = synthesizers.get(model.id);
|
||||
if (cached) return cached;
|
||||
const s = loadSherpa();
|
||||
const p = (f: string) => path.join(model.dir, f);
|
||||
let ttsModel: Record<string, unknown>;
|
||||
switch (model.engine) {
|
||||
case 'kokoro':
|
||||
ttsModel = {
|
||||
kokoro: {
|
||||
model: p('model.onnx'),
|
||||
voices: p('voices.bin'),
|
||||
tokens: p('tokens.txt'),
|
||||
dataDir: p('espeak-ng-data'),
|
||||
lexicon: [p('lexicon-us-en.txt'), p('lexicon-zh.txt')].join(',')
|
||||
}
|
||||
};
|
||||
break;
|
||||
case 'piper': {
|
||||
const onnx = model.files.find((f) => f.endsWith('.onnx')) ?? 'model.onnx';
|
||||
ttsModel = { vits: { model: p(onnx), tokens: p('tokens.txt'), dataDir: p('espeak-ng-data') } };
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw new Error(`Moteur TTS non supporté : ${model.engine}`);
|
||||
}
|
||||
const tts = new s.OfflineTts({ model: { ...ttsModel, numThreads: threads, provider: 'cpu', debug: 0 }, maxNumSentences: 1 });
|
||||
synthesizers.set(model.id, tts);
|
||||
return tts;
|
||||
}
|
||||
|
||||
// ── audio helpers ──────────────────────────────────────────────────────────
|
||||
function decodeWav(bytes: Uint8Array): { samples: Float32Array; sampleRate: number } {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
if (String.fromCharCode(bytes[0], bytes[1], bytes[2], bytes[3]) !== 'RIFF') throw new Error('WAV invalide');
|
||||
let offset = 12;
|
||||
let sampleRate = 16000;
|
||||
let channels = 1;
|
||||
let bits = 16;
|
||||
let data: { start: number; length: number } | null = null;
|
||||
while (offset + 8 <= bytes.byteLength) {
|
||||
const id = String.fromCharCode(bytes[offset], bytes[offset + 1], bytes[offset + 2], bytes[offset + 3]);
|
||||
const size = view.getUint32(offset + 4, true);
|
||||
if (id === 'fmt ') {
|
||||
channels = view.getUint16(offset + 10, true);
|
||||
sampleRate = view.getUint32(offset + 12, true);
|
||||
bits = view.getUint16(offset + 22, true);
|
||||
} else if (id === 'data') {
|
||||
data = { start: offset + 8, length: Math.min(size, bytes.byteLength - offset - 8) };
|
||||
break;
|
||||
}
|
||||
offset += 8 + size + (size % 2);
|
||||
}
|
||||
if (!data) throw new Error('WAV sans données');
|
||||
const bytesPerSample = bits / 8;
|
||||
const frames = Math.floor(data.length / bytesPerSample / channels);
|
||||
const samples = new Float32Array(frames);
|
||||
for (let i = 0; i < frames; i++) {
|
||||
let sum = 0;
|
||||
for (let c = 0; c < channels; c++) {
|
||||
const pos = data.start + (i * channels + c) * bytesPerSample;
|
||||
sum += bits === 16 ? view.getInt16(pos, true) / 32768 : bits === 32 ? view.getInt32(pos, true) / 2147483648 : (bytes[pos] - 128) / 128;
|
||||
}
|
||||
samples[i] = sum / channels;
|
||||
}
|
||||
return { samples, sampleRate };
|
||||
}
|
||||
|
||||
function resampleTo16k(samples: Float32Array, rate: number): Float32Array {
|
||||
if (rate === 16000) return samples;
|
||||
const ratio = rate / 16000;
|
||||
const out = new Float32Array(Math.round(samples.length / ratio));
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
const pos = i * ratio;
|
||||
const idx = Math.floor(pos);
|
||||
const frac = pos - idx;
|
||||
const a = samples[Math.min(idx, samples.length - 1)];
|
||||
const b = samples[Math.min(idx + 1, samples.length - 1)];
|
||||
out[i] = a + (b - a) * frac;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function encodeWav(samples: Float32Array, sampleRate: number): Uint8Array {
|
||||
const buffer = new ArrayBuffer(44 + samples.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
const str = (o: number, s: string) => {
|
||||
for (let i = 0; i < s.length; i++) view.setUint8(o + i, s.charCodeAt(i));
|
||||
};
|
||||
str(0, 'RIFF');
|
||||
view.setUint32(4, 36 + samples.length * 2, true);
|
||||
str(8, 'WAVE');
|
||||
str(12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * 2, true);
|
||||
view.setUint16(32, 2, true);
|
||||
view.setUint16(34, 16, true);
|
||||
str(36, 'data');
|
||||
view.setUint32(40, samples.length * 2, true);
|
||||
let o = 44;
|
||||
for (let i = 0; i < samples.length; i++, o += 2) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]));
|
||||
view.setInt16(o, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
}
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
// ── request handling ───────────────────────────────────────────────────────
|
||||
function handle(req: Request): unknown {
|
||||
switch (req.type) {
|
||||
case 'status': {
|
||||
try {
|
||||
const s = loadSherpa();
|
||||
return { available: true, version: s.version, loaded: [...recognizers.keys(), ...synthesizers.keys()] };
|
||||
} catch (err) {
|
||||
return { available: false, error: (err as Error).message, loaded: [] };
|
||||
}
|
||||
}
|
||||
case 'transcribe': {
|
||||
const started = Date.now();
|
||||
const { samples, sampleRate } = decodeWav(req.wav);
|
||||
const pcm = resampleTo16k(samples, sampleRate);
|
||||
const recognizer = getRecognizer(req.model, req.language);
|
||||
const stream = recognizer.createStream();
|
||||
stream.acceptWaveform({ sampleRate: 16000, samples: pcm });
|
||||
recognizer.decode(stream);
|
||||
const result = recognizer.getResult(stream);
|
||||
return { text: (result.text ?? '').trim(), language: result.lang, durationMs: Date.now() - started, audioSec: pcm.length / 16000 };
|
||||
}
|
||||
case 'synthesize': {
|
||||
const started = Date.now();
|
||||
const tts = getSynthesizer(req.model);
|
||||
const sid = Math.max(0, Math.min(tts.numSpeakers - 1, Math.floor(req.speaker)));
|
||||
const audio = tts.generate({ text: req.text, sid, speed: Math.max(0.5, Math.min(2, req.speed || 1)) });
|
||||
return { wav: encodeWav(audio.samples, audio.sampleRate), sampleRate: audio.sampleRate, durationMs: Date.now() - started, audioSec: audio.samples.length / audio.sampleRate };
|
||||
}
|
||||
case 'unload': {
|
||||
if (req.modelId) {
|
||||
for (const k of [...recognizers.keys()]) if (k.startsWith(`${req.modelId}:`)) recognizers.delete(k);
|
||||
synthesizers.delete(req.modelId);
|
||||
} else {
|
||||
recognizers.clear();
|
||||
synthesizers.clear();
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
default:
|
||||
throw new Error('requête inconnue');
|
||||
}
|
||||
}
|
||||
|
||||
function respond(req: Request): Response {
|
||||
try {
|
||||
return { id: req.id, ok: true, result: handle(req) };
|
||||
} catch (err) {
|
||||
return { id: req.id, ok: false, error: (err as Error).message || String(err) };
|
||||
}
|
||||
}
|
||||
|
||||
// Electron utility process transport, with a child_process fallback for tests.
|
||||
const parentPort = (process as unknown as { parentPort?: { on: (ev: 'message', cb: (e: { data: Request }) => void) => void; postMessage: (m: unknown) => void } }).parentPort;
|
||||
if (parentPort) {
|
||||
parentPort.on('message', (event) => parentPort.postMessage(respond(event.data)));
|
||||
} else if (process.send) {
|
||||
process.on('message', (msg: Request) => process.send!(respond(msg)));
|
||||
}
|
||||
Reference in new issue
Block a user