mirror of
https://github.com/R0m1k3/EveFlow.git
synced 2026-10-23 23:28:14 +02:00
feat!: rebuild EveFlow as a JARVIS-style voice HUD for Hermes Agent (v2.0.0)
Complete rewrite of the application: - Toolchain: Electron 44, Vite 8, React 19, TypeScript 5.9, Zustand, Vitest; main process rewritten in TypeScript (electron/), typed IPC contract (shared/), sandboxed renderer with webSecurity on and an HTTP/SSE proxy in the main process. - Voice: AudioWorklet microphone capture at 16 kHz with adaptive energy VAD and auto-stop, WAV encoder, OpenAI-compatible STT, TTS queue with sentence-level streaming, prefetch and Web Audio playback feeding an analyser; hands-free mode, global push-to-talk hotkey, system-voice fallback. - Hermes: full API client (capabilities, health, models, skills, toolsets, sessions, jobs) with automatic transport selection: runs API (SSE lifecycle, approvals, steer, stop) > sessions stream > chat completions with local tools; tolerant event normalisation; webhook receiver hardened (secret, size limit). - UI: JARVIS arc-reactor core on Canvas 2D reacting to the real audio signal, holographic HUD layout (transcript, core, Hermes ops panel with tools/crons/ skills/sessions/link tabs, telemetry), settings drawer, approval modals, compact floating widget, four themes, tray icon and global shortcuts. - Removed the 3D Eve robot, three.js and legacy assets; migrated 1.x settings. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y
This commit is contained in:
239 files changed
+11351
-86495
No files matched your search
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Shared Web Audio graph. Everything that is played (TTS) or captured (mic) passes through
|
||||
* analysers so the JARVIS core can react to the real signal.
|
||||
*/
|
||||
class AudioBus {
|
||||
private ctx: AudioContext | null = null;
|
||||
private outputGain: GainNode | null = null;
|
||||
private outputAnalyser: AnalyserNode | null = null;
|
||||
private inputAnalyser: AnalyserNode | null = null;
|
||||
private timeData = new Float32Array(1024);
|
||||
private freqData = new Uint8Array(512);
|
||||
private synthetic = 0;
|
||||
|
||||
get context(): AudioContext {
|
||||
if (!this.ctx) {
|
||||
this.ctx = new AudioContext({ latencyHint: 'interactive' });
|
||||
this.outputGain = this.ctx.createGain();
|
||||
this.outputAnalyser = this.ctx.createAnalyser();
|
||||
this.outputAnalyser.fftSize = 1024;
|
||||
this.outputAnalyser.smoothingTimeConstant = 0.6;
|
||||
this.outputGain.connect(this.outputAnalyser);
|
||||
this.outputAnalyser.connect(this.ctx.destination);
|
||||
}
|
||||
return this.ctx;
|
||||
}
|
||||
|
||||
async resume(): Promise<void> {
|
||||
const ctx = this.context;
|
||||
if (ctx.state === 'suspended') await ctx.resume().catch(() => undefined);
|
||||
}
|
||||
|
||||
get output(): AudioNode {
|
||||
this.context;
|
||||
return this.outputGain!;
|
||||
}
|
||||
|
||||
setVolume(volume: number): void {
|
||||
this.context;
|
||||
this.outputGain!.gain.value = Math.max(0, Math.min(1.5, volume));
|
||||
}
|
||||
|
||||
/** Attach an external analyser (mic capture) so the HUD can read the input level. */
|
||||
setInputAnalyser(analyser: AnalyserNode | null): void {
|
||||
this.inputAnalyser = analyser;
|
||||
}
|
||||
|
||||
/** For engines that bypass Web Audio (system speech synthesis), fake a plausible level. */
|
||||
setSyntheticLevel(level: number): void {
|
||||
this.synthetic = level;
|
||||
}
|
||||
|
||||
private rms(analyser: AnalyserNode): number {
|
||||
if (this.timeData.length !== analyser.fftSize) this.timeData = new Float32Array(analyser.fftSize);
|
||||
analyser.getFloatTimeDomainData(this.timeData);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < this.timeData.length; i++) sum += this.timeData[i] * this.timeData[i];
|
||||
return Math.sqrt(sum / this.timeData.length);
|
||||
}
|
||||
|
||||
/** Output loudness 0..1 (TTS). */
|
||||
outputLevel(): number {
|
||||
if (this.synthetic > 0) return this.synthetic;
|
||||
if (!this.outputAnalyser) return 0;
|
||||
return Math.min(1, this.rms(this.outputAnalyser) * 4);
|
||||
}
|
||||
|
||||
/** Input loudness 0..1 (microphone). */
|
||||
inputLevel(): number {
|
||||
if (!this.inputAnalyser) return 0;
|
||||
return Math.min(1, this.rms(this.inputAnalyser) * 6);
|
||||
}
|
||||
|
||||
/** Normalised output spectrum (0..1) with `bins` values, low to high frequency. */
|
||||
spectrum(bins: number, source: 'output' | 'input' = 'output'): Float32Array {
|
||||
const analyser = source === 'output' ? this.outputAnalyser : this.inputAnalyser;
|
||||
const out = new Float32Array(bins);
|
||||
if (!analyser) {
|
||||
if (this.synthetic > 0) {
|
||||
const t = performance.now() / 1000;
|
||||
for (let i = 0; i < bins; i++) {
|
||||
out[i] = Math.max(0, this.synthetic * (0.6 + 0.4 * Math.sin(t * 9 + i * 0.7)) * (1 - i / bins));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (this.freqData.length !== analyser.frequencyBinCount) this.freqData = new Uint8Array(analyser.frequencyBinCount);
|
||||
analyser.getByteFrequencyData(this.freqData);
|
||||
// Only the lower half of the spectrum carries voice energy.
|
||||
const usable = Math.floor(this.freqData.length * 0.5);
|
||||
for (let i = 0; i < bins; i++) {
|
||||
const from = Math.floor((i / bins) * usable);
|
||||
const to = Math.max(from + 1, Math.floor(((i + 1) / bins) * usable));
|
||||
let sum = 0;
|
||||
for (let j = from; j < to; j++) sum += this.freqData[j];
|
||||
out[i] = sum / (to - from) / 255;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
export const audioBus = new AudioBus();
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Microphone capture built on AudioWorklet (no deprecated ScriptProcessor, no mic monitoring
|
||||
* feedback). Produces 16 kHz WAV utterances, with energy VAD for hands-free auto stop.
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { audioBus } from './audioBus';
|
||||
import { DEFAULT_VAD, EnergyVad, type VadOptions, type VadSignal } from './vad';
|
||||
import { buildWav16k, rms, type WavResult } from './wav';
|
||||
|
||||
const WORKLET_SOURCE = `
|
||||
class EveFlowCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffer = new Float32Array(2048);
|
||||
this.offset = 0;
|
||||
}
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || input.length === 0) return true;
|
||||
const channel = input[0];
|
||||
if (!channel) return true;
|
||||
let i = 0;
|
||||
while (i < channel.length) {
|
||||
const n = Math.min(channel.length - i, this.buffer.length - this.offset);
|
||||
this.buffer.set(channel.subarray(i, i + n), this.offset);
|
||||
this.offset += n;
|
||||
i += n;
|
||||
if (this.offset === this.buffer.length) {
|
||||
this.port.postMessage(this.buffer, [this.buffer.buffer]);
|
||||
this.buffer = new Float32Array(2048);
|
||||
this.offset = 0;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('eveflow-capture', EveFlowCaptureProcessor);
|
||||
`;
|
||||
|
||||
export type CaptureMode = 'auto' | 'manual';
|
||||
|
||||
export interface CaptureCallbacks {
|
||||
onLevel?: (level: number) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onUtterance: (wav: WavResult) => void;
|
||||
onEnd: (reason: 'stopped' | 'speech-end' | 'no-speech' | 'too-short' | 'max-length' | 'error') => void;
|
||||
onError?: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface CaptureOptions {
|
||||
deviceId?: string;
|
||||
mode: CaptureMode;
|
||||
vad?: Partial<VadOptions>;
|
||||
callbacks: CaptureCallbacks;
|
||||
}
|
||||
|
||||
export interface MicDevice {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
let workletUrl: string | null = null;
|
||||
|
||||
function getWorkletUrl(): string {
|
||||
if (!workletUrl) workletUrl = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'application/javascript' }));
|
||||
return workletUrl;
|
||||
}
|
||||
|
||||
export async function listMicrophones(): Promise<MicDevice[]> {
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
return devices
|
||||
.filter((d) => d.kind === 'audioinput')
|
||||
.map((d, i) => ({ deviceId: d.deviceId, label: d.label || `Microphone ${i + 1}` }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export class MicCapture {
|
||||
private ctx: AudioContext | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private worklet: AudioWorkletNode | null = null;
|
||||
private legacy: ScriptProcessorNode | null = null;
|
||||
private chunks: Float32Array[] = [];
|
||||
private totalSamples = 0;
|
||||
private vad: EnergyVad | null = null;
|
||||
private active = false;
|
||||
private options: CaptureOptions | null = null;
|
||||
private sampleRate = 16_000;
|
||||
|
||||
get isActive(): boolean {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
async start(options: CaptureOptions): Promise<void> {
|
||||
if (this.active) this.stop('stopped');
|
||||
this.options = options;
|
||||
this.chunks = [];
|
||||
this.totalSamples = 0;
|
||||
this.vad = options.mode === 'auto' ? new EnergyVad(options.vad ?? DEFAULT_VAD) : null;
|
||||
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error("Le microphone n'est pas accessible dans cet environnement (getUserMedia indisponible).");
|
||||
}
|
||||
|
||||
const constraints: MediaStreamConstraints = {
|
||||
audio: {
|
||||
deviceId: options.deviceId ? { exact: options.deviceId } : undefined,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
channelCount: 1
|
||||
}
|
||||
};
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
} catch (err) {
|
||||
const e = err as DOMException;
|
||||
const detail =
|
||||
e.name === 'NotAllowedError'
|
||||
? "Accès au micro refusé. Vérifiez les paramètres de confidentialité Windows (Microphone) et relancez EveFlow."
|
||||
: e.name === 'NotFoundError'
|
||||
? 'Aucun microphone détecté.'
|
||||
: e.name === 'NotReadableError'
|
||||
? 'Le microphone est utilisé par une autre application.'
|
||||
: `${e.name}: ${e.message}`;
|
||||
Log.error('mic', 'getUserMedia failed', { name: e.name, message: e.message });
|
||||
throw new Error(detail);
|
||||
}
|
||||
|
||||
try {
|
||||
this.ctx = new AudioContext({ sampleRate: 16_000, latencyHint: 'interactive' });
|
||||
} catch {
|
||||
this.ctx = new AudioContext({ latencyHint: 'interactive' });
|
||||
}
|
||||
if (this.ctx.state === 'suspended') await this.ctx.resume().catch(() => undefined);
|
||||
this.sampleRate = this.ctx.sampleRate;
|
||||
|
||||
const source = this.ctx.createMediaStreamSource(this.stream);
|
||||
const analyser = this.ctx.createAnalyser();
|
||||
analyser.fftSize = 512;
|
||||
analyser.smoothingTimeConstant = 0.5;
|
||||
source.connect(analyser);
|
||||
audioBus.setInputAnalyser(analyser);
|
||||
|
||||
try {
|
||||
await this.ctx.audioWorklet.addModule(getWorkletUrl());
|
||||
this.worklet = new AudioWorkletNode(this.ctx, 'eveflow-capture', { numberOfInputs: 1, numberOfOutputs: 0, channelCount: 1 });
|
||||
this.worklet.port.onmessage = (event: MessageEvent<Float32Array>) => this.onSamples(event.data);
|
||||
source.connect(this.worklet);
|
||||
} catch (err) {
|
||||
Log.warn('mic', `AudioWorklet unavailable, falling back to ScriptProcessor: ${(err as Error).message}`);
|
||||
const processor = this.ctx.createScriptProcessor(2048, 1, 1);
|
||||
processor.onaudioprocess = (e) => this.onSamples(new Float32Array(e.inputBuffer.getChannelData(0)));
|
||||
const mute = this.ctx.createGain();
|
||||
mute.gain.value = 0; // Chrome requires a destination path, but we never want to hear the mic.
|
||||
source.connect(processor);
|
||||
processor.connect(mute);
|
||||
mute.connect(this.ctx.destination);
|
||||
this.legacy = processor;
|
||||
}
|
||||
|
||||
this.active = true;
|
||||
Log.info('mic', `capture started (${this.sampleRate} Hz, mode=${options.mode})`);
|
||||
}
|
||||
|
||||
private onSamples(samples: Float32Array): void {
|
||||
if (!this.active || !this.options) return;
|
||||
this.chunks.push(samples);
|
||||
this.totalSamples += samples.length;
|
||||
const level = rms(samples);
|
||||
this.options.callbacks.onLevel?.(Math.min(1, level * 6));
|
||||
|
||||
if (this.vad) {
|
||||
const frameMs = (samples.length / this.sampleRate) * 1000;
|
||||
const signal = this.vad.feed(level, frameMs);
|
||||
this.handleVad(signal);
|
||||
} else if (this.totalSamples / this.sampleRate > 120) {
|
||||
this.finishUtterance('max-length');
|
||||
}
|
||||
}
|
||||
|
||||
private handleVad(signal: VadSignal): void {
|
||||
switch (signal) {
|
||||
case 'speech-start':
|
||||
this.options?.callbacks.onSpeechStart?.();
|
||||
break;
|
||||
case 'speech-end':
|
||||
case 'max-length':
|
||||
this.finishUtterance(signal);
|
||||
break;
|
||||
case 'too-short':
|
||||
case 'no-speech':
|
||||
this.stop(signal);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private finishUtterance(reason: 'speech-end' | 'max-length' | 'stopped'): void {
|
||||
const opts = this.options;
|
||||
const chunks = this.chunks;
|
||||
const rate = this.sampleRate;
|
||||
this.teardown();
|
||||
if (!opts) return;
|
||||
if (chunks.length === 0) {
|
||||
opts.callbacks.onEnd('no-speech');
|
||||
return;
|
||||
}
|
||||
const wav = buildWav16k(chunks, rate);
|
||||
Log.info('mic', `utterance captured: ${wav.durationSec.toFixed(2)}s, ${wav.bytes.length} bytes`);
|
||||
if (wav.durationSec < 0.25) {
|
||||
opts.callbacks.onEnd('too-short');
|
||||
return;
|
||||
}
|
||||
opts.callbacks.onUtterance(wav);
|
||||
opts.callbacks.onEnd(reason);
|
||||
}
|
||||
|
||||
/** Stop capture. In manual mode this finalises the utterance; otherwise reports the reason. */
|
||||
stop(reason: 'stopped' | 'no-speech' | 'too-short' = 'stopped'): void {
|
||||
if (!this.active) return;
|
||||
if (reason === 'stopped') {
|
||||
this.finishUtterance('stopped');
|
||||
return;
|
||||
}
|
||||
const opts = this.options;
|
||||
this.teardown();
|
||||
opts?.callbacks.onEnd(reason);
|
||||
}
|
||||
|
||||
/** Abort without producing an utterance. */
|
||||
cancel(): void {
|
||||
if (!this.active) return;
|
||||
this.teardown();
|
||||
this.options?.callbacks.onEnd('stopped');
|
||||
}
|
||||
|
||||
private teardown(): void {
|
||||
this.active = false;
|
||||
audioBus.setInputAnalyser(null);
|
||||
if (this.worklet) {
|
||||
this.worklet.port.onmessage = null;
|
||||
this.worklet.disconnect();
|
||||
this.worklet = null;
|
||||
}
|
||||
if (this.legacy) {
|
||||
this.legacy.onaudioprocess = null;
|
||||
this.legacy.disconnect();
|
||||
this.legacy = null;
|
||||
}
|
||||
this.stream?.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
void this.ctx?.close().catch(() => undefined);
|
||||
this.ctx = null;
|
||||
this.chunks = [];
|
||||
this.totalSamples = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Singleton facade over the TTS engine bound to the settings store, exposing speaking state
|
||||
* to the voice and chat stores.
|
||||
*/
|
||||
import { useChat } from '../../state/chat';
|
||||
import { useSettings } from '../../state/settings';
|
||||
import { useVoice } from '../../state/voice';
|
||||
import { TtsEngine } from './tts';
|
||||
|
||||
class SpeechFacade {
|
||||
private engine: TtsEngine | null = null;
|
||||
private streaming = false;
|
||||
private streamEnabled = true;
|
||||
|
||||
private get tts(): TtsEngine {
|
||||
if (!this.engine) {
|
||||
const settings = useSettings.getState().settings;
|
||||
this.engine = new TtsEngine(settings.speech);
|
||||
this.engine.onState((state) => {
|
||||
useVoice.getState().setTts(state);
|
||||
const chat = useChat.getState();
|
||||
if (state === 'speaking') chat.setHud('speaking');
|
||||
else if (state === 'idle' && chat.hud === 'speaking') chat.setHud(chat.isSending ? 'thinking' : 'idle');
|
||||
});
|
||||
useSettings.subscribe((s) => this.engine?.updateConfig(s.settings.speech));
|
||||
}
|
||||
return this.engine;
|
||||
}
|
||||
|
||||
init(): void {
|
||||
this.tts;
|
||||
}
|
||||
|
||||
isSpeaking(): boolean {
|
||||
return this.engine?.isActive ?? false;
|
||||
}
|
||||
|
||||
say(text: string): void {
|
||||
if (useSettings.getState().settings.speech.provider === 'off') return;
|
||||
this.tts.speak(text);
|
||||
}
|
||||
|
||||
pushStream(delta: string): void {
|
||||
if (!useSettings.getState().settings.speech.autoSpeak) return;
|
||||
this.streaming = true;
|
||||
if (this.streamEnabled) this.tts.pushStream(delta);
|
||||
}
|
||||
|
||||
endStream(): void {
|
||||
if (this.streaming) this.tts.endStream();
|
||||
this.streaming = false;
|
||||
}
|
||||
|
||||
discardStream(): void {
|
||||
this.streaming = false;
|
||||
this.tts.stop();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.streaming = false;
|
||||
this.engine?.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export const speech = new SpeechFacade();
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Log } from '../../lib/log';
|
||||
import { httpFetch } from '../../lib/transport';
|
||||
|
||||
export type SttProvider = 'openai-compatible' | 'browser';
|
||||
|
||||
export interface SttConfig {
|
||||
provider: SttProvider;
|
||||
apiUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
language: string; // BCP-47, e.g. fr-FR
|
||||
prompt?: string;
|
||||
}
|
||||
|
||||
export function sttEndpoint(apiUrl: string): string {
|
||||
const base = apiUrl.trim().replace(/\/+$/, '');
|
||||
if (base.endsWith('/transcriptions')) return base;
|
||||
if (base.endsWith('/audio')) return `${base}/transcriptions`;
|
||||
if (/\/v\d+$/.test(base)) return `${base}/audio/transcriptions`;
|
||||
return `${base}/v1/audio/transcriptions`;
|
||||
}
|
||||
|
||||
/** Transcribe a WAV buffer through any OpenAI-compatible `/v1/audio/transcriptions` endpoint. */
|
||||
export async function transcribeWav(wav: Uint8Array, config: SttConfig): Promise<string> {
|
||||
if (!config.apiUrl.trim()) throw new Error("URL de l'API de transcription non configurée.");
|
||||
const endpoint = sttEndpoint(config.apiUrl);
|
||||
const fields: Record<string, string> = {
|
||||
model: config.model || 'whisper-1',
|
||||
response_format: 'json'
|
||||
};
|
||||
if (config.language) fields.language = config.language.split('-')[0];
|
||||
if (config.prompt) fields.prompt = config.prompt;
|
||||
const headers: Record<string, string> = {};
|
||||
if (config.apiKey.trim()) headers.Authorization = `Bearer ${config.apiKey.trim()}`;
|
||||
|
||||
Log.info('stt', `POST ${endpoint}`, { model: fields.model, bytes: wav.length });
|
||||
const res = await httpFetch({
|
||||
url: endpoint,
|
||||
method: 'POST',
|
||||
headers,
|
||||
multipart: { fields, file: { name: 'audio.wav', type: 'audio/wav', data: wav } },
|
||||
timeoutMs: 90_000
|
||||
});
|
||||
if (!res.ok) {
|
||||
let detail = res.text ?? '';
|
||||
try {
|
||||
const parsed = JSON.parse(detail) as { error?: { message?: string }; message?: string; detail?: string };
|
||||
detail = parsed.error?.message ?? parsed.message ?? parsed.detail ?? detail;
|
||||
} catch {
|
||||
/* raw text */
|
||||
}
|
||||
throw new Error(`Transcription refusée (HTTP ${res.status}) : ${detail.slice(0, 240)}`);
|
||||
}
|
||||
const text = res.text ?? '';
|
||||
try {
|
||||
const parsed = JSON.parse(text) as { text?: string; transcript?: string; segments?: Array<{ text: string }> };
|
||||
const value = parsed.text ?? parsed.transcript ?? parsed.segments?.map((s) => s.text).join(' ');
|
||||
if (typeof value === 'string') return value.trim();
|
||||
} catch {
|
||||
if (text.trim()) return text.trim();
|
||||
}
|
||||
throw new Error('Réponse de transcription invalide.');
|
||||
}
|
||||
|
||||
/** Web Speech API fallback (Chromium routes it to Google; works only online, quality varies). */
|
||||
export class BrowserRecognizer {
|
||||
private recognition: SpeechRecognition | null = null;
|
||||
|
||||
static isSupported(): boolean {
|
||||
return typeof window !== 'undefined' && !!(window.SpeechRecognition || window.webkitSpeechRecognition);
|
||||
}
|
||||
|
||||
start(lang: string, onResult: (text: string) => void, onError: (message: string) => void, onEnd: () => void): void {
|
||||
const Ctor = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||
if (!Ctor) {
|
||||
onError("La reconnaissance vocale du navigateur n'est pas disponible.");
|
||||
return;
|
||||
}
|
||||
this.stop();
|
||||
const rec = new Ctor();
|
||||
rec.lang = lang;
|
||||
rec.continuous = false;
|
||||
rec.interimResults = false;
|
||||
rec.maxAlternatives = 1;
|
||||
rec.onresult = (event) => {
|
||||
const text = event.results[0]?.[0]?.transcript ?? '';
|
||||
onResult(text);
|
||||
};
|
||||
rec.onerror = (event) => onError(event.error === 'not-allowed' ? 'Micro refusé par le navigateur.' : `Reconnaissance: ${event.error}`);
|
||||
rec.onend = () => {
|
||||
this.recognition = null;
|
||||
onEnd();
|
||||
};
|
||||
this.recognition = rec;
|
||||
try {
|
||||
rec.start();
|
||||
} catch (err) {
|
||||
onError((err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
try {
|
||||
this.recognition?.stop();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
this.recognition = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Text-to-speech engine with a sentence queue, prefetching and Web Audio playback so the HUD
|
||||
* reacts to the actual waveform. Providers: OpenAI-compatible /v1/audio/speech, system voices,
|
||||
* and the legacy Google Translate endpoint (no key, online only).
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { chunkForSpeech, cleanForSpeech, extractSentences } from '../../lib/text';
|
||||
import { httpFetch } from '../../lib/transport';
|
||||
import { audioBus } from './audioBus';
|
||||
|
||||
export type TtsProvider = 'openai-compatible' | 'system' | 'google-free' | 'off';
|
||||
|
||||
export interface TtsConfig {
|
||||
provider: TtsProvider;
|
||||
apiUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
voice: string;
|
||||
speed: number; // 0.5 - 2
|
||||
format: 'mp3' | 'wav' | 'opus';
|
||||
systemVoice: string;
|
||||
language: string;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
export type TtsState = 'idle' | 'loading' | 'speaking';
|
||||
|
||||
interface QueueItem {
|
||||
text: string;
|
||||
audio?: Promise<AudioBuffer | null>;
|
||||
}
|
||||
|
||||
export function ttsEndpoint(apiUrl: string): string {
|
||||
const base = apiUrl.trim().replace(/\/+$/, '');
|
||||
if (base.endsWith('/speech')) return base;
|
||||
if (base.endsWith('/audio')) return `${base}/speech`;
|
||||
if (/\/v\d+$/.test(base)) return `${base}/audio/speech`;
|
||||
return `${base}/v1/audio/speech`;
|
||||
}
|
||||
|
||||
export class TtsEngine {
|
||||
private queue: QueueItem[] = [];
|
||||
private playing = false;
|
||||
private currentSource: AudioBufferSourceNode | null = null;
|
||||
private currentUtterance: SpeechSynthesisUtterance | null = null;
|
||||
private streamBuffer = '';
|
||||
private generation = 0;
|
||||
private config: TtsConfig;
|
||||
private listeners = new Set<(state: TtsState) => void>();
|
||||
private state: TtsState = 'idle';
|
||||
|
||||
constructor(config: TtsConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
updateConfig(config: TtsConfig): void {
|
||||
this.config = config;
|
||||
audioBus.setVolume(config.volume);
|
||||
}
|
||||
|
||||
onState(listener: (state: TtsState) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
get isActive(): boolean {
|
||||
return this.playing || this.queue.length > 0;
|
||||
}
|
||||
|
||||
private setState(state: TtsState): void {
|
||||
if (this.state === state) return;
|
||||
this.state = state;
|
||||
for (const l of this.listeners) l(state);
|
||||
}
|
||||
|
||||
/** Speak a finished text (clears any pending speech first). */
|
||||
speak(text: string): void {
|
||||
this.stop();
|
||||
for (const chunk of chunkForSpeech(text)) this.enqueue(chunk);
|
||||
}
|
||||
|
||||
/** Feed streamed tokens; complete sentences are spoken as soon as they are available. */
|
||||
pushStream(delta: string): void {
|
||||
this.streamBuffer += delta;
|
||||
const { sentences, rest } = extractSentences(this.streamBuffer);
|
||||
this.streamBuffer = rest;
|
||||
for (const sentence of sentences) this.enqueue(sentence);
|
||||
}
|
||||
|
||||
/** Flush the remainder of a streamed message. */
|
||||
endStream(): void {
|
||||
const rest = this.streamBuffer.trim();
|
||||
this.streamBuffer = '';
|
||||
if (rest) for (const chunk of chunkForSpeech(rest)) this.enqueue(chunk);
|
||||
}
|
||||
|
||||
enqueue(text: string): void {
|
||||
if (this.config.provider === 'off') return;
|
||||
const clean = cleanForSpeech(text);
|
||||
if (!clean || !/[\p{L}\p{N}]/u.test(clean)) return;
|
||||
const item: QueueItem = { text: clean };
|
||||
if (this.config.provider !== 'system') item.audio = this.prefetch(clean, this.generation);
|
||||
this.queue.push(item);
|
||||
void this.drain();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.generation++;
|
||||
this.queue = [];
|
||||
this.streamBuffer = '';
|
||||
if (this.currentSource) {
|
||||
try {
|
||||
this.currentSource.onended = null;
|
||||
this.currentSource.stop();
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
this.currentSource = null;
|
||||
}
|
||||
if (this.currentUtterance) {
|
||||
this.currentUtterance.onend = null;
|
||||
this.currentUtterance = null;
|
||||
}
|
||||
if (typeof speechSynthesis !== 'undefined') speechSynthesis.cancel();
|
||||
audioBus.setSyntheticLevel(0);
|
||||
this.playing = false;
|
||||
this.setState('idle');
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
if (this.playing) return;
|
||||
this.playing = true;
|
||||
const gen = this.generation;
|
||||
while (this.queue.length > 0 && gen === this.generation) {
|
||||
const item = this.queue.shift()!;
|
||||
try {
|
||||
if (item.audio) {
|
||||
this.setState('loading');
|
||||
const buffer = await item.audio;
|
||||
if (gen !== this.generation) break;
|
||||
if (buffer) {
|
||||
this.setState('speaking');
|
||||
await this.playBuffer(buffer, gen);
|
||||
} else {
|
||||
await this.speakSystem(item.text, gen);
|
||||
}
|
||||
} else {
|
||||
this.setState('speaking');
|
||||
await this.speakSystem(item.text, gen);
|
||||
}
|
||||
} catch (err) {
|
||||
Log.warn('tts', `segment failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
if (gen === this.generation) {
|
||||
this.playing = false;
|
||||
this.setState('idle');
|
||||
}
|
||||
}
|
||||
|
||||
private prefetch(text: string, gen: number): Promise<AudioBuffer | null> {
|
||||
const task = this.config.provider === 'google-free' ? this.fetchGoogle(text) : this.fetchOpenAi(text);
|
||||
return task
|
||||
.then(async (bytes) => {
|
||||
if (gen !== this.generation || !bytes) return null;
|
||||
await audioBus.resume();
|
||||
const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
||||
return audioBus.context.decodeAudioData(copy);
|
||||
})
|
||||
.catch((err) => {
|
||||
Log.warn('tts', `synthesis failed, falling back to system voice: ${(err as Error).message}`);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private async fetchOpenAi(text: string): Promise<Uint8Array> {
|
||||
const { apiUrl, apiKey, model, voice, speed, format } = this.config;
|
||||
if (!apiUrl.trim()) throw new Error('URL TTS non configurée');
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (apiKey.trim()) headers.Authorization = `Bearer ${apiKey.trim()}`;
|
||||
const res = await httpFetch({
|
||||
url: ttsEndpoint(apiUrl),
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
model: model || 'tts-1',
|
||||
input: text,
|
||||
voice: voice || 'alloy',
|
||||
speed: Math.max(0.5, Math.min(2, speed || 1)),
|
||||
response_format: format || 'mp3'
|
||||
}),
|
||||
responseType: 'binary',
|
||||
timeoutMs: 60_000
|
||||
});
|
||||
if (!res.ok || !res.binary) {
|
||||
const body = res.binary ? new TextDecoder().decode(res.binary).slice(0, 200) : '';
|
||||
throw new Error(`HTTP ${res.status} ${body}`);
|
||||
}
|
||||
return res.binary;
|
||||
}
|
||||
|
||||
private async fetchGoogle(text: string): Promise<Uint8Array> {
|
||||
const url = `https://translate.google.com/translate_tts?ie=UTF-8&tl=${encodeURIComponent(this.config.language.split('-')[0] || 'fr')}&client=tw-ob&q=${encodeURIComponent(text.slice(0, 200))}`;
|
||||
const res = await httpFetch({
|
||||
url,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
|
||||
Referer: 'https://translate.google.com/'
|
||||
},
|
||||
responseType: 'binary',
|
||||
timeoutMs: 15_000
|
||||
});
|
||||
if (!res.ok || !res.binary) throw new Error(`Google TTS HTTP ${res.status}`);
|
||||
return res.binary;
|
||||
}
|
||||
|
||||
private playBuffer(buffer: AudioBuffer, gen: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (gen !== this.generation) return resolve();
|
||||
const ctx = audioBus.context;
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
if (this.config.provider === 'google-free') source.playbackRate.value = Math.max(0.5, Math.min(2, this.config.speed || 1));
|
||||
source.connect(audioBus.output);
|
||||
source.onended = () => {
|
||||
if (this.currentSource === source) this.currentSource = null;
|
||||
resolve();
|
||||
};
|
||||
this.currentSource = source;
|
||||
source.start();
|
||||
});
|
||||
}
|
||||
|
||||
private speakSystem(text: string, gen: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (typeof speechSynthesis === 'undefined' || gen !== this.generation) return resolve();
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.lang = this.config.language || 'fr-FR';
|
||||
utterance.rate = Math.max(0.5, Math.min(2, this.config.speed || 1));
|
||||
utterance.pitch = 1;
|
||||
utterance.volume = Math.max(0, Math.min(1, this.config.volume));
|
||||
const voice = this.resolveSystemVoice();
|
||||
if (voice) utterance.voice = voice;
|
||||
let pulse: ReturnType<typeof setInterval> | null = setInterval(() => {
|
||||
audioBus.setSyntheticLevel(0.35 + Math.random() * 0.4);
|
||||
}, 90);
|
||||
const finish = () => {
|
||||
if (pulse) clearInterval(pulse);
|
||||
pulse = null;
|
||||
audioBus.setSyntheticLevel(0);
|
||||
if (this.currentUtterance === utterance) this.currentUtterance = null;
|
||||
resolve();
|
||||
};
|
||||
utterance.onend = finish;
|
||||
utterance.onerror = finish;
|
||||
this.currentUtterance = utterance;
|
||||
speechSynthesis.speak(utterance);
|
||||
});
|
||||
}
|
||||
|
||||
private resolveSystemVoice(): SpeechSynthesisVoice | undefined {
|
||||
if (typeof speechSynthesis === 'undefined') return undefined;
|
||||
const voices = speechSynthesis.getVoices();
|
||||
if (this.config.systemVoice) {
|
||||
const exact = voices.find((v) => v.name === this.config.systemVoice);
|
||||
if (exact) return exact;
|
||||
}
|
||||
const lang = (this.config.language || 'fr').toLowerCase().split('-')[0];
|
||||
const candidates = voices.filter((v) => v.lang.toLowerCase().startsWith(lang));
|
||||
return candidates.sort((a, b) => scoreVoice(b) - scoreVoice(a))[0] ?? voices[0];
|
||||
}
|
||||
}
|
||||
|
||||
export function scoreVoice(v: SpeechSynthesisVoice): number {
|
||||
const name = v.name.toLowerCase();
|
||||
let score = 0;
|
||||
if (name.includes('natural') || name.includes('neural') || name.includes('online')) score += 40;
|
||||
if (name.includes('google')) score += 30;
|
||||
if (name.includes('microsoft')) score += 10;
|
||||
if (v.localService) score += 5;
|
||||
return score;
|
||||
}
|
||||
|
||||
export function listSystemVoices(): SpeechSynthesisVoice[] {
|
||||
if (typeof speechSynthesis === 'undefined') return [];
|
||||
return [...speechSynthesis.getVoices()].sort((a, b) => a.lang.localeCompare(b.lang) || scoreVoice(b) - scoreVoice(a));
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Energy-based voice activity detector with an adaptive noise floor.
|
||||
* Pure logic (no Web Audio) so it can be unit tested.
|
||||
*/
|
||||
export interface VadOptions {
|
||||
/** Signal must exceed noiseFloor * ratio (and absolute minimum) to count as speech. */
|
||||
speechRatio: number;
|
||||
minRms: number;
|
||||
/** Silence duration that ends an utterance. */
|
||||
silenceMs: number;
|
||||
/** Minimum duration of speech before it is considered a real utterance. */
|
||||
minSpeechMs: number;
|
||||
/** Hard cap on one utterance. */
|
||||
maxUtteranceMs: number;
|
||||
/** Give up if nobody speaks for this long after starting. */
|
||||
noSpeechTimeoutMs: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_VAD: VadOptions = {
|
||||
speechRatio: 2.6,
|
||||
minRms: 0.012,
|
||||
silenceMs: 900,
|
||||
minSpeechMs: 250,
|
||||
maxUtteranceMs: 45_000,
|
||||
noSpeechTimeoutMs: 9_000
|
||||
};
|
||||
|
||||
export type VadSignal = 'idle' | 'speech-start' | 'speaking' | 'speech-end' | 'too-short' | 'no-speech' | 'max-length';
|
||||
|
||||
export class EnergyVad {
|
||||
private noiseFloor: number;
|
||||
private speaking = false;
|
||||
private speechMs = 0;
|
||||
private silenceMs = 0;
|
||||
private elapsedMs = 0;
|
||||
private calibrationMs = 0;
|
||||
readonly options: VadOptions;
|
||||
|
||||
constructor(options: Partial<VadOptions> = {}) {
|
||||
this.options = { ...DEFAULT_VAD, ...options };
|
||||
this.noiseFloor = this.options.minRms;
|
||||
}
|
||||
|
||||
get isSpeaking(): boolean {
|
||||
return this.speaking;
|
||||
}
|
||||
|
||||
get floor(): number {
|
||||
return this.noiseFloor;
|
||||
}
|
||||
|
||||
/** Feed one analysis frame. Returns the transition that occurred. */
|
||||
feed(frameRms: number, frameMs: number): VadSignal {
|
||||
const o = this.options;
|
||||
this.elapsedMs += frameMs;
|
||||
const threshold = Math.max(o.minRms, this.noiseFloor * o.speechRatio);
|
||||
const loud = frameRms > threshold;
|
||||
|
||||
// Adaptive noise floor: fast during the first 400 ms, slow afterwards, only while not speaking.
|
||||
if (!loud || this.calibrationMs < 400) {
|
||||
const alpha = this.calibrationMs < 400 ? 0.25 : 0.02;
|
||||
this.noiseFloor = this.noiseFloor * (1 - alpha) + frameRms * alpha;
|
||||
this.calibrationMs += frameMs;
|
||||
}
|
||||
|
||||
if (!this.speaking) {
|
||||
if (loud) {
|
||||
this.speechMs += frameMs;
|
||||
if (this.speechMs >= Math.min(120, o.minSpeechMs)) {
|
||||
this.speaking = true;
|
||||
this.silenceMs = 0;
|
||||
return 'speech-start';
|
||||
}
|
||||
} else {
|
||||
this.speechMs = 0;
|
||||
if (this.elapsedMs >= o.noSpeechTimeoutMs) return 'no-speech';
|
||||
}
|
||||
return 'idle';
|
||||
}
|
||||
|
||||
// speaking
|
||||
this.speechMs += frameMs;
|
||||
if (loud) this.silenceMs = 0;
|
||||
else this.silenceMs += frameMs;
|
||||
|
||||
if (this.speechMs >= o.maxUtteranceMs) return this.finish('max-length');
|
||||
if (this.silenceMs >= o.silenceMs) {
|
||||
const spoken = this.speechMs - this.silenceMs;
|
||||
return this.finish(spoken >= o.minSpeechMs ? 'speech-end' : 'too-short');
|
||||
}
|
||||
return 'speaking';
|
||||
}
|
||||
|
||||
private finish(signal: VadSignal): VadSignal {
|
||||
this.speaking = false;
|
||||
this.speechMs = 0;
|
||||
this.silenceMs = 0;
|
||||
return signal;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.speaking = false;
|
||||
this.speechMs = 0;
|
||||
this.silenceMs = 0;
|
||||
this.elapsedMs = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
/**
|
||||
* Voice controller: microphone → VAD → STT → conversation, plus hands-free loop and barge-in.
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { useChat } from '../../state/chat';
|
||||
import { useSettings } from '../../state/settings';
|
||||
import { useVoice } from '../../state/voice';
|
||||
import { sendMessage } from '../conversation';
|
||||
import { audioBus } from './audioBus';
|
||||
import { listMicrophones, MicCapture } from './capture';
|
||||
import { speech } from './speech';
|
||||
import { BrowserRecognizer, transcribeWav } from './stt';
|
||||
import type { WavResult } from './wav';
|
||||
|
||||
const SENSITIVITY_RATIO: Record<number, number> = { 1: 4.5, 2: 3.4, 3: 2.6, 4: 2.0, 5: 1.6 };
|
||||
const SENSITIVITY_MIN_RMS: Record<number, number> = { 1: 0.03, 2: 0.02, 3: 0.012, 4: 0.008, 5: 0.005 };
|
||||
|
||||
class VoiceController {
|
||||
private capture = new MicCapture();
|
||||
private browser = new BrowserRecognizer();
|
||||
private handsFreeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private stopping = false;
|
||||
private unsubscribeVoice: (() => void) | null = null;
|
||||
|
||||
init(): void {
|
||||
void listMicrophones().then((devices) => useVoice.getState().setMicDevices(devices));
|
||||
// Hands-free: re-arm the microphone once the assistant stops speaking.
|
||||
this.unsubscribeVoice = useVoice.subscribe((state, prev) => {
|
||||
if (prev.tts !== 'idle' && state.tts === 'idle' && state.handsFree && state.phase === 'off') {
|
||||
this.scheduleHandsFree(350);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.unsubscribeVoice?.();
|
||||
this.stop();
|
||||
}
|
||||
|
||||
get isListening(): boolean {
|
||||
return useVoice.getState().phase !== 'off';
|
||||
}
|
||||
|
||||
toggle(): void {
|
||||
if (this.isListening) this.stop();
|
||||
else void this.start();
|
||||
}
|
||||
|
||||
setHandsFree(on: boolean): void {
|
||||
useVoice.getState().setHandsFree(on);
|
||||
useSettings.getState().update({ voice: { handsFree: on } });
|
||||
if (on && !this.isListening && !speech.isSpeaking()) void this.start();
|
||||
if (!on) {
|
||||
this.clearHandsFreeTimer();
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleHandsFree(delay: number): void {
|
||||
this.clearHandsFreeTimer();
|
||||
this.handsFreeTimer = setTimeout(() => {
|
||||
const v = useVoice.getState();
|
||||
if (v.handsFree && v.phase === 'off' && !useChat.getState().isSending && !speech.isSpeaking()) void this.start();
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private clearHandsFreeTimer(): void {
|
||||
if (this.handsFreeTimer) clearTimeout(this.handsFreeTimer);
|
||||
this.handsFreeTimer = null;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
const voice = useVoice.getState();
|
||||
if (voice.phase !== 'off') return;
|
||||
const settings = useSettings.getState().settings.voice;
|
||||
voice.setError(null);
|
||||
voice.setInterim('');
|
||||
if (!settings.bargeIn) speech.stop();
|
||||
else speech.stop();
|
||||
useChat.getState().setHud('listening');
|
||||
voice.setPhase('arming');
|
||||
|
||||
if (settings.provider === 'browser') {
|
||||
this.startBrowser(settings.language);
|
||||
return;
|
||||
}
|
||||
|
||||
const sensitivity = Math.min(5, Math.max(1, Math.round(settings.sensitivity))) as 1 | 2 | 3 | 4 | 5;
|
||||
try {
|
||||
await audioBus.resume();
|
||||
await this.capture.start({
|
||||
deviceId: settings.micDeviceId || undefined,
|
||||
mode: settings.captureMode,
|
||||
vad: {
|
||||
silenceMs: settings.silenceMs,
|
||||
speechRatio: SENSITIVITY_RATIO[sensitivity],
|
||||
minRms: SENSITIVITY_MIN_RMS[sensitivity],
|
||||
noSpeechTimeoutMs: useVoice.getState().handsFree ? 20_000 : 9_000
|
||||
},
|
||||
callbacks: {
|
||||
onLevel: (level) => useVoice.getState().setInputLevel(level),
|
||||
onSpeechStart: () => useVoice.getState().setPhase('speech'),
|
||||
onUtterance: (wav) => void this.transcribe(wav),
|
||||
onEnd: (reason) => this.onCaptureEnd(reason),
|
||||
onError: (message) => useVoice.getState().setError(message)
|
||||
}
|
||||
});
|
||||
useVoice.getState().setPhase('listening');
|
||||
this.playChime(true);
|
||||
} catch (err) {
|
||||
const message = (err as Error).message;
|
||||
useVoice.getState().setError(message);
|
||||
useVoice.getState().setPhase('off');
|
||||
useChat.getState().setHud('error');
|
||||
useChat.getState().setError(message);
|
||||
Log.error('voice', message);
|
||||
setTimeout(() => useChat.getState().setHud('idle'), 2500);
|
||||
}
|
||||
}
|
||||
|
||||
private startBrowser(lang: string): void {
|
||||
useVoice.getState().setPhase('listening');
|
||||
this.browser.start(
|
||||
lang,
|
||||
(text) => {
|
||||
useVoice.getState().setTranscript(text);
|
||||
if (text.trim()) void sendMessage(text, [], 'voice');
|
||||
},
|
||||
(message) => {
|
||||
useVoice.getState().setError(message);
|
||||
useChat.getState().setError(message);
|
||||
},
|
||||
() => this.onCaptureEnd('stopped')
|
||||
);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopping = true;
|
||||
this.clearHandsFreeTimer();
|
||||
if (useSettings.getState().settings.voice.provider === 'browser') this.browser.stop();
|
||||
else if (this.capture.isActive) this.capture.stop('stopped');
|
||||
else this.onCaptureEnd('stopped');
|
||||
this.stopping = false;
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.clearHandsFreeTimer();
|
||||
this.browser.stop();
|
||||
this.capture.cancel();
|
||||
}
|
||||
|
||||
private onCaptureEnd(reason: string): void {
|
||||
const voice = useVoice.getState();
|
||||
voice.setInputLevel(0);
|
||||
if (voice.phase !== 'transcribing') voice.setPhase('off');
|
||||
const chat = useChat.getState();
|
||||
if (chat.hud === 'listening') chat.setHud(chat.isSending ? 'thinking' : 'idle');
|
||||
if (reason === 'no-speech') {
|
||||
if (voice.handsFree && !this.stopping) this.scheduleHandsFree(400);
|
||||
else voice.setError("Aucune parole détectée.");
|
||||
} else if (reason === 'too-short') {
|
||||
if (voice.handsFree && !this.stopping) this.scheduleHandsFree(200);
|
||||
}
|
||||
}
|
||||
|
||||
private async transcribe(wav: WavResult): Promise<void> {
|
||||
const voice = useVoice.getState();
|
||||
voice.setPhase('transcribing');
|
||||
useChat.getState().setHud('thinking');
|
||||
this.playChime(false);
|
||||
const settings = useSettings.getState().settings.voice;
|
||||
try {
|
||||
const text = await transcribeWav(wav.bytes, settings);
|
||||
voice.setTranscript(text);
|
||||
voice.setPhase('off');
|
||||
Log.info('voice', `transcript (${wav.durationSec.toFixed(1)}s): ${text}`);
|
||||
if (text.trim()) {
|
||||
await sendMessage(text, [], 'voice');
|
||||
} else {
|
||||
voice.setError('Transcription vide.');
|
||||
useChat.getState().setHud('idle');
|
||||
if (voice.handsFree) this.scheduleHandsFree(300);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = (err as Error).message;
|
||||
voice.setError(message);
|
||||
voice.setPhase('off');
|
||||
useChat.getState().setError(message);
|
||||
useChat.getState().setHud('error');
|
||||
Log.error('voice', `transcription failed: ${message}`);
|
||||
setTimeout(() => useChat.getState().setHud('idle'), 3000);
|
||||
}
|
||||
}
|
||||
|
||||
private playChime(up: boolean): void {
|
||||
if (!useSettings.getState().settings.voice.wakeChime) return;
|
||||
try {
|
||||
const ctx = audioBus.context;
|
||||
const osc = ctx.createOscillator();
|
||||
const gain = ctx.createGain();
|
||||
osc.type = 'sine';
|
||||
const now = ctx.currentTime;
|
||||
osc.frequency.setValueAtTime(up ? 660 : 880, now);
|
||||
osc.frequency.exponentialRampToValueAtTime(up ? 990 : 550, now + 0.12);
|
||||
gain.gain.setValueAtTime(0.0001, now);
|
||||
gain.gain.exponentialRampToValueAtTime(0.08, now + 0.02);
|
||||
gain.gain.exponentialRampToValueAtTime(0.0001, now + 0.16);
|
||||
osc.connect(gain);
|
||||
gain.connect(audioBus.output);
|
||||
osc.start(now);
|
||||
osc.stop(now + 0.18);
|
||||
} catch {
|
||||
/* audio unavailable */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const voiceController = new VoiceController();
|
||||
@@ -0,0 +1,106 @@
|
||||
/** PCM utilities: merge, resample and encode 16-bit mono WAV. */
|
||||
|
||||
export function mergeChunks(chunks: Float32Array[], totalLength?: number): Float32Array {
|
||||
const length = totalLength ?? chunks.reduce((n, c) => n + c.length, 0);
|
||||
const out = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
if (offset + chunk.length > length) {
|
||||
out.set(chunk.subarray(0, length - offset), offset);
|
||||
break;
|
||||
}
|
||||
out.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Linear-interpolation resampler with a box pre-filter when downsampling. */
|
||||
export function resample(input: Float32Array, fromRate: number, toRate: number): Float32Array {
|
||||
if (fromRate === toRate || input.length === 0) return input;
|
||||
const ratio = fromRate / toRate;
|
||||
const outLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Float32Array(outLength);
|
||||
if (ratio > 1) {
|
||||
for (let i = 0; i < outLength; i++) {
|
||||
const start = Math.floor(i * ratio);
|
||||
const end = Math.min(input.length, Math.floor((i + 1) * ratio));
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
for (let j = start; j < end; j++) {
|
||||
sum += input[j];
|
||||
count++;
|
||||
}
|
||||
out[i] = count > 0 ? sum / count : input[Math.min(start, input.length - 1)];
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < outLength; i++) {
|
||||
const pos = i * ratio;
|
||||
const idx = Math.floor(pos);
|
||||
const frac = pos - idx;
|
||||
const a = input[Math.min(idx, input.length - 1)];
|
||||
const b = input[Math.min(idx + 1, input.length - 1)];
|
||||
out[i] = a + (b - a) * frac;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function rms(samples: Float32Array): number {
|
||||
if (samples.length === 0) return 0;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i];
|
||||
return Math.sqrt(sum / samples.length);
|
||||
}
|
||||
|
||||
/** Peak-normalise quiet recordings so ASR models get a healthy signal. */
|
||||
export function normalizeGain(samples: Float32Array, targetPeak = 0.9): Float32Array {
|
||||
let peak = 0;
|
||||
for (let i = 0; i < samples.length; i++) peak = Math.max(peak, Math.abs(samples[i]));
|
||||
if (peak === 0 || peak >= targetPeak * 0.8) return samples;
|
||||
const gain = Math.min(8, targetPeak / peak);
|
||||
const out = new Float32Array(samples.length);
|
||||
for (let i = 0; i < samples.length; i++) out[i] = samples[i] * gain;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function encodeWav(samples: Float32Array, sampleRate: number): Uint8Array {
|
||||
const buffer = new ArrayBuffer(44 + samples.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
const writeString = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i));
|
||||
};
|
||||
writeString(0, 'RIFF');
|
||||
view.setUint32(4, 36 + samples.length * 2, true);
|
||||
writeString(8, 'WAVE');
|
||||
writeString(12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * 2, true);
|
||||
view.setUint16(32, 2, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeString(36, 'data');
|
||||
view.setUint32(40, samples.length * 2, true);
|
||||
let offset = 44;
|
||||
for (let i = 0; i < samples.length; i++, offset += 2) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]));
|
||||
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
}
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
export interface WavResult {
|
||||
bytes: Uint8Array;
|
||||
sampleRate: number;
|
||||
durationSec: number;
|
||||
}
|
||||
|
||||
/** Build a 16 kHz mono WAV (the format every OpenAI-compatible ASR accepts) from captured chunks. */
|
||||
export function buildWav16k(chunks: Float32Array[], captureRate: number): WavResult {
|
||||
const merged = mergeChunks(chunks);
|
||||
const resampled = resample(merged, captureRate, 16_000);
|
||||
const normalized = normalizeGain(resampled);
|
||||
return { bytes: encodeWav(normalized, 16_000), sampleRate: 16_000, durationSec: normalized.length / 16_000 };
|
||||
}
|
||||
Reference in new issue
Block a user