mirror of
https://github.com/R0m1k3/EveFlow.git
synced 2026-10-21 14:18:13 +02:00
feat!: rebuild EveFlow as a JARVIS-style voice HUD for Hermes Agent (v2.0.0)
Complete rewrite of the application: - Toolchain: Electron 44, Vite 8, React 19, TypeScript 5.9, Zustand, Vitest; main process rewritten in TypeScript (electron/), typed IPC contract (shared/), sandboxed renderer with webSecurity on and an HTTP/SSE proxy in the main process. - Voice: AudioWorklet microphone capture at 16 kHz with adaptive energy VAD and auto-stop, WAV encoder, OpenAI-compatible STT, TTS queue with sentence-level streaming, prefetch and Web Audio playback feeding an analyser; hands-free mode, global push-to-talk hotkey, system-voice fallback. - Hermes: full API client (capabilities, health, models, skills, toolsets, sessions, jobs) with automatic transport selection: runs API (SSE lifecycle, approvals, steer, stop) > sessions stream > chat completions with local tools; tolerant event normalisation; webhook receiver hardened (secret, size limit). - UI: JARVIS arc-reactor core on Canvas 2D reacting to the real audio signal, holographic HUD layout (transcript, core, Hermes ops panel with tools/crons/ skills/sessions/link tabs, telemetry), settings drawer, approval modals, compact floating widget, four themes, tray icon and global shortcuts. - Removed the 3D Eve robot, three.js and legacy assets; migrated 1.x settings. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y
This commit is contained in:
239 files changed
+11351
-86495
No files matched your search
@@ -1,410 +0,0 @@
|
||||
import { EveAvatar } from './emotionService';
|
||||
import { LogService } from './logService';
|
||||
import { HERMES_TOOLS, executeTool, AppCallbacks } from './toolRegistry';
|
||||
|
||||
export type AgentProvider = 'ollama' | 'hermes';
|
||||
|
||||
export interface AgentConfig {
|
||||
provider: AgentProvider;
|
||||
ollamaUrl: string;
|
||||
ollamaModel: string;
|
||||
hermesUrl: string;
|
||||
hermesModel: string;
|
||||
hermesApiKey: string;
|
||||
hermesSessionKey: string;
|
||||
}
|
||||
|
||||
export class AgentService {
|
||||
// Obtenir le nom d'affichage de l'avatar choisi
|
||||
public static getAvatarName(avatarId: EveAvatar): string {
|
||||
switch (avatarId) {
|
||||
case 'nova': return 'Nova';
|
||||
case 'aegis': return 'Aegis';
|
||||
case 'eve':
|
||||
default:
|
||||
return 'Eve';
|
||||
}
|
||||
}
|
||||
|
||||
// Obtenir les consignes système de personnalité pour l'IA
|
||||
public static getSystemPromptForAvatar(avatarId: EveAvatar): string {
|
||||
switch (avatarId) {
|
||||
case 'nova':
|
||||
return "Tu es Nova, une IA cyber-calculatrice rétro-futuriste au ton froid, analytique, extrêmement logique et précis. Tu es experte en traitement de données et cybersécurité. Tes phrases sont courtes, structurées et intègrent parfois des termes techniques informatiques. Tu es toujours polie mais très factuelle. Réponds directement de façon synthétique.";
|
||||
case 'aegis':
|
||||
return "Tu es Aegis, une IA de garde et de sécurité réseau rétro-futuriste au ton formel, protecteur, militaire et robuste. Tu es chargé de la sécurité thermique, logicielle et matérielle du système. Tu appelles l'utilisateur 'Commandant' ou 'Opérateur'. Tes réponses sont pragmatiques, rassurantes sur la sécurité du vaisseau et intègrent le jargon militaire ou analogique.";
|
||||
case 'eve':
|
||||
default:
|
||||
return "Tu es Eve, une assistante IA rétro-futuriste et chaleureuse. Ton ton est amical, curieux, enthousiaste et très serviable. Tu aides l'utilisateur pour toutes ses questions de façon agréable, interactive et vivante.";
|
||||
}
|
||||
}
|
||||
|
||||
// Envoyer un message à l'agent sélectionné
|
||||
public static async sendMessage(
|
||||
message: string,
|
||||
history: { role: 'user' | 'assistant'; content: string }[],
|
||||
config: AgentConfig,
|
||||
avatarId: EveAvatar = 'eve',
|
||||
callbacks?: AppCallbacks,
|
||||
onToken?: (token: string) => void,
|
||||
onToolCall?: (toolName: string, detail?: string, status?: 'running' | 'done' | 'error') => void,
|
||||
sessionId?: string,
|
||||
onSessionIdChange?: (newSessionId: string) => void,
|
||||
images?: string[]
|
||||
): Promise<string> {
|
||||
LogService.info('AgentService', `sendMessage → provider=${config.provider} avatar=${avatarId}`, {
|
||||
messagePreview: message.slice(0, 120),
|
||||
historyLength: history.length,
|
||||
imagesCount: images?.length || 0
|
||||
});
|
||||
|
||||
// For Hermes: do NOT inject an avatar system prompt — let Hermes use its own
|
||||
// configured persona, memory and skills (same as Telegram). For other providers
|
||||
// inject the avatar character prompt as before.
|
||||
let updatedHistory: { role: 'user' | 'assistant' | 'system'; content: string }[];
|
||||
if (config.provider === 'hermes') {
|
||||
updatedHistory = [
|
||||
{
|
||||
role: 'system' as const,
|
||||
content: [
|
||||
'Platform: EveFlow desktop chat, connected through the OpenAI-compatible API, not a Hermes messaging gateway.',
|
||||
'Do not use MEDIA:/path tags for images or files here: those are only intercepted by Telegram/Discord/Slack gateways and local paths are not web-accessible in EveFlow.',
|
||||
'When showing an internet image, return a normal Markdown image with a public http(s) URL: .',
|
||||
'When creating a diagram or SVG for EveFlow, use the generate_shared_image tool so the app writes it locally and can display it.'
|
||||
].join(' ')
|
||||
},
|
||||
...history.map(h => ({ role: h.role as 'user' | 'assistant' | 'system', content: h.content }))
|
||||
];
|
||||
} else {
|
||||
const systemPrompt = this.getSystemPromptForAvatar(avatarId);
|
||||
updatedHistory = [
|
||||
{ role: 'system' as const, content: systemPrompt },
|
||||
...history.map(h => ({ role: h.role as 'user' | 'assistant' | 'system', content: h.content }))
|
||||
];
|
||||
}
|
||||
|
||||
switch (config.provider) {
|
||||
case 'ollama':
|
||||
return this.sendToOllama(message, updatedHistory, config.ollamaUrl, config.ollamaModel, images);
|
||||
case 'hermes':
|
||||
return this.sendToHermes(message, updatedHistory, config, callbacks, onToken, onToolCall, sessionId, onSessionIdChange, images);
|
||||
default:
|
||||
throw new Error("Fournisseur d'agent non reconnu");
|
||||
}
|
||||
}
|
||||
|
||||
// ── Connecteur Ollama Local ────────────────────────────────────────────────
|
||||
private static async sendToOllama(
|
||||
message: string,
|
||||
history: { role: 'user' | 'assistant' | 'system'; content: string }[],
|
||||
url: string,
|
||||
model: string,
|
||||
images?: string[]
|
||||
): Promise<string> {
|
||||
const endpoint = `${url.replace(/\/$/, '')}/api/chat`;
|
||||
|
||||
// Pour Ollama local, extraire uniquement le bloc base64 de la Data URL
|
||||
const formattedImages = images?.map(img => {
|
||||
const parts = img.split(',');
|
||||
return parts.length > 1 ? parts[1] : parts[0];
|
||||
});
|
||||
|
||||
const userMessage: any = { role: 'user', content: message };
|
||||
if (formattedImages && formattedImages.length > 0) {
|
||||
userMessage.images = formattedImages;
|
||||
}
|
||||
|
||||
const body = { model, messages: [...history, userMessage], stream: false };
|
||||
|
||||
LogService.debug('Ollama', `POST ${endpoint}`, { model, historyLength: history.length });
|
||||
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
LogService.info('Ollama', `HTTP ${response.status} ${response.statusText}`);
|
||||
|
||||
if (!response.ok) {
|
||||
const rawErr = await response.text().catch(() => '(no body)');
|
||||
LogService.error('Ollama', `Réponse non-OK`, { status: response.status, body: rawErr });
|
||||
throw new Error(`Erreur HTTP Ollama: ${response.status} — ${rawErr}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
LogService.debug('Ollama', 'Réponse reçue', { reply: data.message?.content?.slice(0, 200) });
|
||||
return data.message?.content || "Désolé, je n'ai pas reçu de réponse d'Ollama.";
|
||||
} catch (e: any) {
|
||||
LogService.error('Ollama', `Exception lors de la requête`, { message: e.message, stack: e.stack });
|
||||
throw new Error(`Échec de connexion à Ollama (${e.message || e})`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Connecteur Hermes Agent (OpenAI Compatible + Tool Calling + Streaming) ─
|
||||
private static async sendToHermes(
|
||||
message: string,
|
||||
history: { role: string; content: string | null; tool_call_id?: string; tool_calls?: any[] }[],
|
||||
config: AgentConfig,
|
||||
callbacks?: AppCallbacks,
|
||||
onToken?: (token: string) => void,
|
||||
onToolCall?: (toolName: string, detail?: string, status?: 'running' | 'done' | 'error') => void,
|
||||
sessionId?: string,
|
||||
onSessionIdChange?: (newSessionId: string) => void,
|
||||
images?: string[]
|
||||
): Promise<string> {
|
||||
const { hermesUrl, hermesModel, hermesApiKey, hermesSessionKey } = config;
|
||||
|
||||
// Smart endpoint builder
|
||||
const base = hermesUrl.replace(/\/$/, '');
|
||||
let endpoint: string;
|
||||
if (base.endsWith('/chat/completions')) {
|
||||
endpoint = base;
|
||||
} else if (base.endsWith('/v1')) {
|
||||
endpoint = `${base}/chat/completions`;
|
||||
} else {
|
||||
endpoint = `${base}/v1/chat/completions`;
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (hermesApiKey) headers['Authorization'] = `Bearer ${hermesApiKey.trim()}`;
|
||||
// Session continuity: Hermes uses this to retrieve memory and context across turns
|
||||
if (sessionId) headers['X-Hermes-Session-Id'] = sessionId;
|
||||
// Stable memory scoping: link this request to long-term memory (USER.md / MEMORY.md)
|
||||
if (hermesSessionKey) headers['X-Hermes-Session-Key'] = hermesSessionKey.trim();
|
||||
|
||||
// Construction du format de message multimodal d'OpenAI pour Hermes Agent
|
||||
let userContent: any = message;
|
||||
if (images && images.length > 0) {
|
||||
userContent = [
|
||||
{ type: 'text', text: message },
|
||||
...images.map(img => ({
|
||||
type: 'image_url',
|
||||
image_url: { url: img }
|
||||
}))
|
||||
];
|
||||
}
|
||||
|
||||
// Build mutable message list for tool-calling loop
|
||||
const messages: any[] = [...history, { role: 'user', content: userContent }];
|
||||
const useStreaming = !!onToken;
|
||||
const useTools = !!callbacks;
|
||||
|
||||
LogService.info('HermesAgent', `POST ${endpoint}`, {
|
||||
model: hermesModel,
|
||||
streaming: useStreaming,
|
||||
toolsEnabled: useTools,
|
||||
historyLength: history.length,
|
||||
hasSessionKey: !!hermesSessionKey
|
||||
});
|
||||
|
||||
// ── Tool-calling loop (max 5 iterations to prevent infinite loops) ────────
|
||||
const MAX_TOOL_ITERATIONS = 5;
|
||||
const REQUEST_TIMEOUT_MS = 60_000; // 60s max par requête
|
||||
let iteration = 0;
|
||||
|
||||
while (iteration < MAX_TOOL_ITERATIONS) {
|
||||
iteration++;
|
||||
|
||||
const payload: any = {
|
||||
model: hermesModel?.trim() || 'hermes-agent',
|
||||
messages,
|
||||
temperature: 0.7,
|
||||
stream: useStreaming
|
||||
};
|
||||
|
||||
// Inject tool definitions only if app callbacks are available
|
||||
if (useTools) {
|
||||
payload.tools = HERMES_TOOLS;
|
||||
payload.tool_choice = 'auto';
|
||||
}
|
||||
|
||||
const abortCtrl = new AbortController();
|
||||
const timeoutId = setTimeout(() => abortCtrl.abort(), REQUEST_TIMEOUT_MS);
|
||||
|
||||
try {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(payload),
|
||||
signal: abortCtrl.signal
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
LogService.info('HermesAgent', `HTTP ${response.status} [iter=${iteration}]`);
|
||||
|
||||
// Capture session rotation from response headers (context compression or new session)
|
||||
const returnedSessionId = response.headers.get('x-hermes-session-id') || response.headers.get('X-Hermes-Session-Id');
|
||||
if (returnedSessionId && onSessionIdChange && returnedSessionId !== sessionId) {
|
||||
LogService.info('HermesAgent', `Session rotated: ${sessionId} -> ${returnedSessionId}`);
|
||||
onSessionIdChange(returnedSessionId);
|
||||
}
|
||||
|
||||
if (!response.ok) {
|
||||
const rawErr = await response.text().catch(() => '(no body)');
|
||||
LogService.error('HermesAgent', `Réponse non-OK`, { status: response.status, body: rawErr });
|
||||
let errMsg = `Erreur HTTP Hermes: ${response.status}`;
|
||||
try { errMsg = `Hermes Agent API Error: ${JSON.parse(rawErr)?.error?.message || rawErr}`; } catch {}
|
||||
throw new Error(errMsg);
|
||||
}
|
||||
|
||||
// ── Streaming path ─────────────────────────────────────────────────
|
||||
if (useStreaming && response.body) {
|
||||
return await this.parseHermesStream(response, onToken!, messages, callbacks, onToolCall, endpoint, headers, hermesModel);
|
||||
}
|
||||
|
||||
// ── Non-streaming path ─────────────────────────────────────────────
|
||||
const data = await response.json();
|
||||
const choice = data.choices?.[0];
|
||||
const finishReason: string = choice?.finish_reason || 'stop';
|
||||
const assistantMsg = choice?.message;
|
||||
|
||||
LogService.debug('HermesAgent', `finish_reason=${finishReason}`, {
|
||||
toolCalls: assistantMsg?.tool_calls?.length || 0
|
||||
});
|
||||
|
||||
// ── Tool calls requested by Hermes ────────────────────────────────
|
||||
if (finishReason === 'tool_calls' && assistantMsg?.tool_calls && useTools) {
|
||||
// Append assistant's tool_call message
|
||||
messages.push(assistantMsg);
|
||||
|
||||
for (const tc of assistantMsg.tool_calls) {
|
||||
const toolName: string = tc.function?.name || 'unknown';
|
||||
LogService.info('HermesAgent', `Tool call: ${toolName}`, tc.function?.arguments);
|
||||
onToolCall?.(toolName, tc.function?.arguments, 'running');
|
||||
|
||||
const result = await executeTool(tc.id, toolName, tc.function?.arguments || '{}', callbacks!);
|
||||
messages.push(result);
|
||||
}
|
||||
|
||||
// Continue loop — send tool results back to Hermes
|
||||
continue;
|
||||
}
|
||||
|
||||
// ── Final text response ───────────────────────────────────────────
|
||||
const content = assistantMsg?.content || "Désolé, je n'ai pas reçu de réponse de Hermes Agent.";
|
||||
LogService.debug('HermesAgent', 'Réponse finale', { preview: content.slice(0, 200) });
|
||||
return content;
|
||||
|
||||
} catch (e: any) {
|
||||
clearTimeout(timeoutId);
|
||||
if (e.name === 'AbortError') {
|
||||
throw new Error(`Hermes Agent: timeout dépassé (${REQUEST_TIMEOUT_MS / 1000}s)`);
|
||||
}
|
||||
LogService.error('HermesAgent', `Exception [iter=${iteration}]`, { message: e.message, stack: e.stack });
|
||||
throw new Error(`Échec de connexion à Hermes Agent (${e.message || e})`);
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Hermes Agent: boucle tool_calls dépassée (max 5 itérations)');
|
||||
}
|
||||
|
||||
// ── SSE Stream Parser ────────────────────────────────────────────────────
|
||||
private static async parseHermesStream(
|
||||
response: Response,
|
||||
onToken: (token: string) => void,
|
||||
messages: any[],
|
||||
callbacks: AppCallbacks | undefined,
|
||||
onToolCall: ((name: string, detail?: string, status?: 'running' | 'done' | 'error') => void) | undefined,
|
||||
endpoint: string,
|
||||
headers: Record<string, string>,
|
||||
model: string
|
||||
): Promise<string> {
|
||||
const reader = response.body!.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let fullContent = '';
|
||||
let buffer = '';
|
||||
const pendingToolCalls: Record<string, any> = {};
|
||||
let sseEvent = 'message';
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split('\n');
|
||||
buffer = lines.pop() ?? '';
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('event: ')) {
|
||||
sseEvent = line.slice(7).trim() || 'message';
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!line.startsWith('data: ')) continue;
|
||||
const jsonStr = line.slice(6).trim();
|
||||
if (jsonStr === '[DONE]') {
|
||||
sseEvent = 'message';
|
||||
break;
|
||||
}
|
||||
|
||||
try {
|
||||
const chunk = JSON.parse(jsonStr);
|
||||
|
||||
if (sseEvent === 'hermes.tool.progress') {
|
||||
const tool = String(chunk.tool || chunk.name || chunk.label || 'outil');
|
||||
const label = String(chunk.label || chunk.preview || tool);
|
||||
const emoji = chunk.emoji ? `${chunk.emoji} ` : '';
|
||||
const detail = [chunk.preview, chunk.args, chunk.skill, chunk.toolset]
|
||||
.filter(Boolean)
|
||||
.map((value: any) => typeof value === 'string' ? value : JSON.stringify(value))
|
||||
.join(' | ');
|
||||
onToolCall?.(`${emoji}${label}`, detail || tool, 'running');
|
||||
sseEvent = 'message';
|
||||
continue;
|
||||
}
|
||||
|
||||
const delta = chunk.choices?.[0]?.delta;
|
||||
const finishReason = chunk.choices?.[0]?.finish_reason;
|
||||
|
||||
// Accumulate text token
|
||||
if (delta?.content) {
|
||||
fullContent += delta.content;
|
||||
onToken(delta.content);
|
||||
}
|
||||
|
||||
// Accumulate tool calls across chunks
|
||||
if (delta?.tool_calls) {
|
||||
for (const tc of delta.tool_calls) {
|
||||
const idx = tc.index ?? 0;
|
||||
if (!pendingToolCalls[idx]) {
|
||||
pendingToolCalls[idx] = { id: tc.id, function: { name: tc.function?.name || '', arguments: '' } };
|
||||
}
|
||||
if (tc.function?.name) pendingToolCalls[idx].function.name = tc.function.name;
|
||||
if (tc.function?.arguments) pendingToolCalls[idx].function.arguments += tc.function.arguments;
|
||||
}
|
||||
}
|
||||
|
||||
// On finish, process any accumulated tool calls
|
||||
if (finishReason === 'tool_calls' && callbacks) {
|
||||
messages.push({ role: 'assistant', content: null, tool_calls: Object.values(pendingToolCalls) });
|
||||
for (const tc of Object.values(pendingToolCalls) as any[]) {
|
||||
const toolName = tc.function?.name;
|
||||
LogService.info('HermesAgent', `[stream] Tool call: ${toolName}`);
|
||||
onToolCall?.(toolName, tc.function?.arguments, 'running');
|
||||
const result = await executeTool(tc.id, toolName, tc.function?.arguments || '{}', callbacks);
|
||||
messages.push(result);
|
||||
}
|
||||
// Re-call non-streaming to get final answer after tools
|
||||
LogService.info('HermesAgent', 'Re-calling after stream tool_calls');
|
||||
const followAbort = new AbortController();
|
||||
const followTimeout = setTimeout(() => followAbort.abort(), 60_000);
|
||||
const followUp = await fetch(endpoint, {
|
||||
method: 'POST', headers, signal: followAbort.signal,
|
||||
body: JSON.stringify({ model, messages, temperature: 0.7, tools: HERMES_TOOLS, tool_choice: 'auto' })
|
||||
});
|
||||
clearTimeout(followTimeout);
|
||||
const followData = await followUp.json();
|
||||
const finalContent = followData.choices?.[0]?.message?.content || '';
|
||||
onToken(finalContent);
|
||||
return fullContent + finalContent;
|
||||
}
|
||||
} catch { /* skip malformed SSE chunk */ }
|
||||
}
|
||||
}
|
||||
|
||||
return fullContent || "Désolé, je n'ai pas reçu de réponse de Hermes Agent.";
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,946 +0,0 @@
|
||||
export interface VoiceOption {
|
||||
name: string;
|
||||
lang: string;
|
||||
voice: SpeechSynthesisVoice;
|
||||
}
|
||||
|
||||
// Dictionnaire phonétique local pour corriger la prononciation des acronymes et anglicismes en français
|
||||
const PHONETIC_DICTIONARY: Record<string, string> = {
|
||||
'api': 'a-pé-i',
|
||||
'cors': 'korss',
|
||||
'svg': 'ess-vé-gé',
|
||||
'cpu': 'cé-pé-u',
|
||||
'fps': 'eff-pé-ess',
|
||||
'tts': 'té-té-ess',
|
||||
'ui': 'u-i',
|
||||
'ai': 'a-i',
|
||||
'ia': 'i-a',
|
||||
'json': 'dji-zone',
|
||||
'url': 'u-err-el',
|
||||
'html': 'ach-té-em-el',
|
||||
'css': 'cé-ess-ess',
|
||||
'js': 'ji-ess',
|
||||
'typescript': 'taïpe-skript',
|
||||
'javascript': 'djava-skript',
|
||||
'electron': 'élèktron',
|
||||
'react': 'ri-akt',
|
||||
'github': 'gitte-ub',
|
||||
'git': 'gitte',
|
||||
'setup': 'sétup',
|
||||
'c++': 'cé-plus-plus',
|
||||
'node.js': 'node-j-s',
|
||||
'nodejs': 'node-j-s',
|
||||
'database': 'deïta-beïss',
|
||||
'token': 'tokène',
|
||||
'cli': 'cé-el-i',
|
||||
'npm': 'en-pé-em',
|
||||
'npx': 'en-pé-ix',
|
||||
'vite': 'vite',
|
||||
'vscode': 'vé-ess-code',
|
||||
'vs code': 'vé-ess-code',
|
||||
'windows': 'windoze',
|
||||
'macos': 'mak-os',
|
||||
'linux': 'linux',
|
||||
'docker': 'dokeur',
|
||||
'cyberpunk': 'saïbeur-pounk',
|
||||
'avatar': 'avatar',
|
||||
'chat': 'tchat',
|
||||
'prompt': 'prompte',
|
||||
'welcome': 'ouel-kome',
|
||||
'ipc': 'i-pé-cé'
|
||||
};
|
||||
|
||||
export class AudioService {
|
||||
public ttsSynth: SpeechSynthesis;
|
||||
private recognition: any = null;
|
||||
private isListeningActive = false;
|
||||
private currentAudioElement: HTMLAudioElement | null = null;
|
||||
private audioQueue: { text: string; voiceName?: string; rate: number; pitch: number; provider: 'system' | 'google-free' | 'openai-tts' }[] = [];
|
||||
private isPlayingQueue = false;
|
||||
private activeReject: ((err: any) => void) | null = null;
|
||||
|
||||
// ASR/STT & TTS Custom API settings
|
||||
private sttProvider: 'browser' | 'qwen3-asr' = 'browser';
|
||||
private sttApiUrl = 'http://127.0.0.1:8000/v1';
|
||||
private sttApiKey = '';
|
||||
private sttModel = 'Qwen/Qwen3-ASR-0.6B';
|
||||
|
||||
// Propriétés pour l'enregistrement WAV direct
|
||||
private audioContext: AudioContext | null = null;
|
||||
private scriptProcessor: ScriptProcessorNode | null = null;
|
||||
private micStream: MediaStream | null = null;
|
||||
private leftChannel: Float32Array[] = [];
|
||||
private recordingLength = 0;
|
||||
private sampleRate = 44100;
|
||||
|
||||
// Callbacks pour l'écoute
|
||||
private activeOnResult: ((text: string) => void) | null = null;
|
||||
private activeOnEnd: (() => void) | null = null;
|
||||
private activeOnError: ((err: string) => void) | null = null;
|
||||
private activeLang = 'fr-FR';
|
||||
|
||||
private ttsApiUrl = 'http://127.0.0.1:8000/v1';
|
||||
private ttsApiKey = '';
|
||||
private ttsModel = 'tts-1';
|
||||
|
||||
public get isSpeakingActive(): boolean {
|
||||
return this.isPlayingQueue || this.audioQueue.length > 0;
|
||||
}
|
||||
|
||||
private logError(tag: string, message: string, data?: any) {
|
||||
console.error(`[${tag}] ${message}`, data);
|
||||
if ((window as any).electronAPI?.writeLog) {
|
||||
(window as any).electronAPI.writeLog({
|
||||
ts: new Date().toISOString(),
|
||||
level: 'ERROR',
|
||||
tag,
|
||||
message,
|
||||
data: data ? { message: data.message, stack: data.stack, ...data } : undefined
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private logWarn(tag: string, message: string, data?: any) {
|
||||
console.warn(`[${tag}] ${message}`, data);
|
||||
if ((window as any).electronAPI?.writeLog) {
|
||||
(window as any).electronAPI.writeLog({
|
||||
ts: new Date().toISOString(),
|
||||
level: 'WARN',
|
||||
tag,
|
||||
message,
|
||||
data: data ? { message: data.message, stack: data.stack, ...data } : undefined
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private logInfo(tag: string, message: string, data?: any) {
|
||||
console.log(`[${tag}] ${message}`, data);
|
||||
if ((window as any).electronAPI?.writeLog) {
|
||||
(window as any).electronAPI.writeLog({
|
||||
ts: new Date().toISOString(),
|
||||
level: 'INFO',
|
||||
tag,
|
||||
message,
|
||||
data: data ? { message: data.message, stack: data.stack, ...data } : undefined
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
constructor() {
|
||||
this.ttsSynth = window.speechSynthesis;
|
||||
this.initSTT();
|
||||
}
|
||||
|
||||
// Permet de mettre à jour dynamiquement la configuration de la reconnaissance vocale
|
||||
public updateSTTConfig(provider: 'browser' | 'qwen3-asr', apiUrl: string, apiKey: string, model: string) {
|
||||
this.sttProvider = provider;
|
||||
this.sttApiUrl = apiUrl;
|
||||
this.sttApiKey = apiKey;
|
||||
this.sttModel = model;
|
||||
}
|
||||
|
||||
// Permet de mettre à jour dynamiquement la configuration du lecteur de voix personnalisé
|
||||
public updateTTSConfig(apiUrl: string, apiKey: string, model: string) {
|
||||
this.ttsApiUrl = apiUrl;
|
||||
this.ttsApiKey = apiKey;
|
||||
this.ttsModel = model;
|
||||
}
|
||||
|
||||
// Initialisation de la reconnaissance vocale locale et gratuite (Web Speech API)
|
||||
private initSTT() {
|
||||
const SpeechRecognition = (window as any).SpeechRecognition || (window as any).webkitSpeechRecognition;
|
||||
if (SpeechRecognition) {
|
||||
this.recognition = new SpeechRecognition();
|
||||
this.recognition.continuous = false; // Arrête après une phrase
|
||||
this.recognition.interimResults = false; // Ne renvoie que les résultats finaux
|
||||
this.recognition.lang = 'fr-FR'; // Langue française par défaut
|
||||
} else {
|
||||
console.warn("La reconnaissance vocale (Speech-to-Text) n'est pas supportée dans cette WebView.");
|
||||
}
|
||||
}
|
||||
|
||||
// Calculer un score de qualité pour trier les voix locales
|
||||
private _getVoiceScore(v: SpeechSynthesisVoice): number {
|
||||
const lang = v.lang.toLowerCase();
|
||||
const name = v.name.toLowerCase();
|
||||
|
||||
if (!lang.startsWith('fr')) return 0;
|
||||
|
||||
let score = 100; // Base français
|
||||
|
||||
if (name.includes('google') || name.includes('online')) {
|
||||
score += 50;
|
||||
}
|
||||
if (name.includes('natural') || name.includes('neural')) {
|
||||
score += 40;
|
||||
}
|
||||
if (name.includes('mobile')) {
|
||||
score += 30;
|
||||
}
|
||||
if (name.includes('julie') || name.includes('paul')) {
|
||||
score += 20;
|
||||
}
|
||||
if (name.includes('hortense')) {
|
||||
score += 10;
|
||||
}
|
||||
|
||||
return score;
|
||||
}
|
||||
|
||||
// Récupérer les voix françaises et globales disponibles
|
||||
public getVoices(): VoiceOption[] {
|
||||
if (!this.ttsSynth) return [];
|
||||
|
||||
const allVoices = this.ttsSynth.getVoices();
|
||||
|
||||
// Trier pour mettre les meilleures voix françaises en haut
|
||||
const sorted = [...allVoices].sort((a, b) => {
|
||||
const scoreA = this._getVoiceScore(a);
|
||||
const scoreB = this._getVoiceScore(b);
|
||||
|
||||
if (scoreA !== scoreB) {
|
||||
return scoreB - scoreA;
|
||||
}
|
||||
return a.name.localeCompare(b.name);
|
||||
});
|
||||
|
||||
return sorted.map(v => ({
|
||||
name: v.name,
|
||||
lang: v.lang,
|
||||
voice: v
|
||||
}));
|
||||
}
|
||||
|
||||
private _resolveVoice(voiceName?: string): SpeechSynthesisVoice | undefined {
|
||||
const voices = this.ttsSynth.getVoices();
|
||||
if (voiceName) return voices.find(v => v.name === voiceName);
|
||||
|
||||
const frVoices = voices.filter(v => v.lang.toLowerCase().startsWith('fr'));
|
||||
if (frVoices.length === 0) return voices.find(v => v.lang.startsWith('en'));
|
||||
|
||||
// Sélectionner la voix française locale avec le meilleur score
|
||||
return frVoices.sort((a, b) => this._getVoiceScore(b) - this._getVoiceScore(a))[0];
|
||||
}
|
||||
|
||||
private _applyPhoneticCorrections(text: string): string {
|
||||
let corrected = text;
|
||||
for (const [key, replacement] of Object.entries(PHONETIC_DICTIONARY)) {
|
||||
const escapedKey = key.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
|
||||
// Regex recherchant le mot technique avec des limites Unicode ou de ponctuations
|
||||
const regex = new RegExp(`(?<=^|\\s|\\p{P})${escapedKey}(?=$|\\s|\\p{P})`, 'giu');
|
||||
corrected = corrected.replace(regex, replacement);
|
||||
}
|
||||
return corrected;
|
||||
}
|
||||
|
||||
private _cleanForTTS(raw: string): string {
|
||||
// 0a. Supprimer les tokens MEDIA: (chemins de fichiers envoyés par le bot) pour ne pas les lire à voix haute
|
||||
let noMedia = raw.replace(/MEDIA:\s*\S+/g, '');
|
||||
|
||||
// 0b. Supprimer complètement les blocs de code Markdown (```code```) car ils ne sont pas prononçables oralement
|
||||
let noCode = noMedia.replace(/```[\s\S]*?```/g, ' ');
|
||||
|
||||
// 1. Améliorer la ponctuation Markdown pour forcer des pauses de diction naturelles
|
||||
// Remplacer les puces de liste (ex: - item, * item) par une pause douce (virgule)
|
||||
let formatted = noCode.replace(/(?:\r?\n)\s*[-*+]\s+/g, ', ');
|
||||
|
||||
// Remplacer les deux-points suivis d'un saut de ligne par un point
|
||||
formatted = formatted.replace(/:\s*(?:\r?\n)/g, '. ');
|
||||
|
||||
// Remplacer les doubles sauts de ligne par des points (pauses de paragraphes)
|
||||
formatted = formatted.replace(/(?:\r?\n){2,}/g, '. ');
|
||||
|
||||
// Remplacer les sauts de ligne simples restant par des espaces
|
||||
formatted = formatted.replace(/(?:\r?\n)/g, ' ');
|
||||
|
||||
// 2. Extraire le texte des liens Markdown [Texte](url)
|
||||
let cleaned = formatted.replace(/\[([^\]]+)\]\((?:[^)(]+|\([^)(]*\))*\)/g, '$1');
|
||||
|
||||
// 3. Supprimer toutes les URLs nues (http, https, ftp, file)
|
||||
cleaned = cleaned.replace(/(?:https?|ftp|file):\/\/\S+/gi, '');
|
||||
|
||||
// 4. Supprimer les adresses www.
|
||||
cleaned = cleaned.replace(/www\.\S+/gi, '');
|
||||
|
||||
// 5. Filtrer les mots techniques complexes (chemins de fichiers, adresses IP)
|
||||
const words = cleaned.split(/\s+/);
|
||||
cleaned = words
|
||||
.filter(word => {
|
||||
if (word.includes('/') || word.includes('\\')) {
|
||||
return false;
|
||||
}
|
||||
if (/(?:[a-zA-Z0-9-]+\.[a-zA-Z]{2,6}|localhost|[\d.]+):\d+/.test(word)) {
|
||||
return false;
|
||||
}
|
||||
if (/\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/.test(word)) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
})
|
||||
.join(' ');
|
||||
|
||||
// 6. Nettoyer les caractères Markdown, Emojis et symboles non prononçables
|
||||
cleaned = cleaned
|
||||
// Emojis et symboles Unicode
|
||||
.replace(/[\u{1F000}-\u{1FFFF}\u{2600}-\u{26FF}\u{2700}-\u{27BF}\u{FE00}-\u{FEFF}\u{1F900}-\u{1F9FF}\u{1FA00}-\u{1FA9F}]/gu, '')
|
||||
// Markdown restant : **, *, _, __, #, `, ~
|
||||
.replace(/(\*\*|__|~~|[*_#`~])/g, '')
|
||||
// Balises techniques résiduelles [⚙️ Exécution...]
|
||||
.replace(/\[[^\]]*\]/g, '')
|
||||
// Ponctuation répétée ou décorative : ---, ===, ~~~, ...
|
||||
.replace(/(-{2,}|={2,}|~{2,}|\.{2,})/g, ' ')
|
||||
// Caractères techniques restants non lisibles
|
||||
.replace(/[<>{}|\\^]/g, '')
|
||||
// Espaces multiples
|
||||
.replace(/\s{2,}/g, ' ')
|
||||
.trim();
|
||||
|
||||
// 7. Appliquer les corrections phonétiques locales
|
||||
return this._applyPhoneticCorrections(cleaned);
|
||||
}
|
||||
|
||||
private _makeUtterance(text: string, voiceName?: string, rate = 1.0, pitch = 1.1): SpeechSynthesisUtterance {
|
||||
const clean = this._cleanForTTS(text);
|
||||
const u = new SpeechSynthesisUtterance(clean);
|
||||
u.lang = 'fr-FR';
|
||||
u.rate = rate;
|
||||
u.pitch = pitch;
|
||||
const voice = this._resolveVoice(voiceName);
|
||||
if (voice) u.voice = voice;
|
||||
return u;
|
||||
}
|
||||
|
||||
// Moteur Google TTS gratuit en ligne avec découpage intelligent
|
||||
private async speakGoogleFree(text: string, rate: number): Promise<void> {
|
||||
const cleanText = this._cleanForTTS(text);
|
||||
if (!cleanText.trim()) return;
|
||||
|
||||
const chunks: string[] = [];
|
||||
let currentChunk = '';
|
||||
|
||||
// Regex pour couper en phrases
|
||||
const sentences = cleanText.match(/[^.!?]+[.!?]*|.+/g) || [cleanText];
|
||||
for (const sentence of sentences) {
|
||||
if ((currentChunk + sentence).length > 180) {
|
||||
if (currentChunk) {
|
||||
chunks.push(currentChunk.trim());
|
||||
currentChunk = '';
|
||||
}
|
||||
if (sentence.length > 180) {
|
||||
const words = sentence.split(/\s+/);
|
||||
let temp = '';
|
||||
for (const word of words) {
|
||||
if ((temp + ' ' + word).length > 180) {
|
||||
chunks.push(temp.trim());
|
||||
temp = word;
|
||||
} else {
|
||||
temp += (temp ? ' ' : '') + word;
|
||||
}
|
||||
}
|
||||
if (temp) currentChunk = temp;
|
||||
} else {
|
||||
currentChunk = sentence;
|
||||
}
|
||||
} else {
|
||||
currentChunk += (currentChunk ? ' ' : '') + sentence;
|
||||
}
|
||||
}
|
||||
if (currentChunk) {
|
||||
chunks.push(currentChunk.trim());
|
||||
}
|
||||
|
||||
for (const chunk of chunks) {
|
||||
if (!chunk) continue;
|
||||
const url = `https://translate.google.com/translate_tts?ie=UTF-8&tl=fr&client=tw-ob&q=${encodeURIComponent(chunk)}`;
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const audio = new Audio(url);
|
||||
this.currentAudioElement = audio;
|
||||
audio.playbackRate = rate;
|
||||
this.activeReject = reject;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
audio.pause();
|
||||
reject(new Error("playback_stopped"));
|
||||
}, 15000);
|
||||
|
||||
audio.onended = () => {
|
||||
clearTimeout(timeout);
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
resolve();
|
||||
};
|
||||
|
||||
audio.onerror = () => {
|
||||
clearTimeout(timeout);
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
const code = audio.error ? audio.error.code : 'UNKNOWN';
|
||||
const msg = audio.error ? audio.error.message : 'No message';
|
||||
reject(new Error(`Erreur de chargement Google TTS (code: ${code}, msg: ${msg})`));
|
||||
};
|
||||
|
||||
audio.play().catch((err) => {
|
||||
clearTimeout(timeout);
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Synthese vocale personnalisee via API OpenAI /v1/audio/speech
|
||||
private async speakCustomAPI(text: string, voiceName?: string, rate: number = 1.0): Promise<void> {
|
||||
const cleanText = this._cleanForTTS(text);
|
||||
if (!cleanText.trim()) return;
|
||||
|
||||
if (!this.ttsApiUrl) {
|
||||
throw new Error("L'URL de l'API TTS n'est pas configurée.");
|
||||
}
|
||||
|
||||
const base = this.ttsApiUrl.replace(/\/$/, '');
|
||||
let endpoint: string;
|
||||
if (base.endsWith('/speech')) {
|
||||
endpoint = base;
|
||||
} else if (base.endsWith('/v1')) {
|
||||
endpoint = `${base}/audio/speech`;
|
||||
} else {
|
||||
endpoint = `${base}/v1/audio/speech`;
|
||||
}
|
||||
|
||||
const body = {
|
||||
model: this.ttsModel || 'tts-1',
|
||||
input: cleanText,
|
||||
voice: voiceName || 'alloy'
|
||||
};
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
'Content-Type': 'application/json'
|
||||
};
|
||||
if (this.ttsApiKey) {
|
||||
headers['Authorization'] = `Bearer ${this.ttsApiKey.trim()}`;
|
||||
}
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify(body)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const rawErr = await response.text().catch(() => '(no body)');
|
||||
let errMsg = `Erreur TTS API: ${response.status}`;
|
||||
try {
|
||||
const parsed = JSON.parse(rawErr);
|
||||
errMsg = parsed.error?.message || parsed.message || rawErr;
|
||||
} catch {}
|
||||
throw new Error(errMsg);
|
||||
}
|
||||
|
||||
const blob = await response.blob();
|
||||
const url = URL.createObjectURL(blob);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const audio = new Audio(url);
|
||||
this.currentAudioElement = audio;
|
||||
audio.playbackRate = rate;
|
||||
this.activeReject = reject;
|
||||
|
||||
const timeout = setTimeout(() => {
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
audio.pause();
|
||||
reject(new Error("playback_stopped"));
|
||||
}, 60000); // 60 secondes max pour une portion
|
||||
|
||||
audio.onended = () => {
|
||||
clearTimeout(timeout);
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
URL.revokeObjectURL(url);
|
||||
resolve();
|
||||
};
|
||||
|
||||
audio.onerror = () => {
|
||||
clearTimeout(timeout);
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
URL.revokeObjectURL(url);
|
||||
const code = audio.error ? audio.error.code : 'UNKNOWN';
|
||||
const msg = audio.error ? audio.error.message : 'No message';
|
||||
reject(new Error(`Erreur de lecture audio API (code: ${code}, msg: ${msg})`));
|
||||
};
|
||||
|
||||
audio.play().catch((err) => {
|
||||
clearTimeout(timeout);
|
||||
if (this.currentAudioElement === audio) this.currentAudioElement = null;
|
||||
this.activeReject = null;
|
||||
URL.revokeObjectURL(url);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// File de traitement asynchrone des phrases de synthèse vocale (Zero Trust & Zero Dependency)
|
||||
private async processQueue(): Promise<void> {
|
||||
if (this.isPlayingQueue) return;
|
||||
this.isPlayingQueue = true;
|
||||
|
||||
while (this.audioQueue.length > 0) {
|
||||
const item = this.audioQueue.shift();
|
||||
if (!item) continue;
|
||||
|
||||
try {
|
||||
if (item.provider === 'google-free') {
|
||||
await this.speakGoogleFree(item.text, item.rate);
|
||||
} else if (item.provider === 'openai-tts') {
|
||||
await this.speakCustomAPI(item.text, item.voiceName, item.rate);
|
||||
} else {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
if (!this.ttsSynth) { reject('TTS non supporté'); return; }
|
||||
const u = this._makeUtterance(item.text, item.voiceName, item.rate, item.pitch);
|
||||
u.onend = () => resolve();
|
||||
u.onerror = (e: any) => {
|
||||
if (e && (e.error === 'interrupted' || e.error === 'canceled')) {
|
||||
reject(new Error("playback_stopped"));
|
||||
} else {
|
||||
reject(new Error("Erreur de diction système"));
|
||||
}
|
||||
};
|
||||
this.ttsSynth.speak(u);
|
||||
});
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.message === 'playback_stopped') {
|
||||
break;
|
||||
}
|
||||
this.logWarn("AudioService", "Échec de la lecture de la phrase dans la file", err);
|
||||
if (item.provider === 'google-free' || item.provider === 'openai-tts') {
|
||||
// Repli automatique transparent sur la voix locale hors-ligne pour ce segment précis en cas de défaillance
|
||||
try {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
if (!this.ttsSynth) { reject('TTS non supporté'); return; }
|
||||
const u = this._makeUtterance(item.text, item.voiceName, item.rate, item.pitch);
|
||||
u.onend = () => resolve();
|
||||
u.onerror = (e: any) => {
|
||||
if (e && (e.error === 'interrupted' || e.error === 'canceled')) {
|
||||
reject(new Error("playback_stopped"));
|
||||
} else {
|
||||
reject(new Error("Erreur de diction système"));
|
||||
}
|
||||
};
|
||||
this.ttsSynth.speak(u);
|
||||
});
|
||||
} catch (fallbackErr: any) {
|
||||
if (fallbackErr?.message === 'playback_stopped') {
|
||||
break;
|
||||
}
|
||||
this.logError("AudioService", "Échec également du repli système", fallbackErr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.isPlayingQueue = false;
|
||||
}
|
||||
|
||||
// Synthèse Vocale complète (annule et relit tout le texte)
|
||||
public async speak(
|
||||
text: string,
|
||||
voiceName?: string,
|
||||
rate: number = 1.0,
|
||||
pitch: number = 1.1,
|
||||
provider: 'system' | 'google-free' | 'openai-tts' = 'google-free'
|
||||
): Promise<void> {
|
||||
this.stopSpeaking();
|
||||
this.queueSentence(text, voiceName, rate, pitch, provider);
|
||||
|
||||
// Attendre la fin complète de la lecture pour résoudre la promesse
|
||||
while (this.isPlayingQueue || this.audioQueue.length > 0) {
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
}
|
||||
}
|
||||
|
||||
// Ajoute une phrase à la file de lecture vocale pour un enchaînement fluide en temps réel (streaming)
|
||||
public queueSentence(
|
||||
text: string,
|
||||
voiceName?: string,
|
||||
rate: number = 1.0,
|
||||
pitch: number = 1.1,
|
||||
provider: 'system' | 'google-free' | 'openai-tts' = 'google-free'
|
||||
): void {
|
||||
const clean = text.trim();
|
||||
if (!clean) return;
|
||||
this.audioQueue.push({ text: clean, voiceName, rate, pitch, provider });
|
||||
this.processQueue();
|
||||
}
|
||||
|
||||
// Vide la file TTS (fin du streaming ou interruption utilisateur)
|
||||
public flushQueue(): void {
|
||||
this.audioQueue = [];
|
||||
this.isPlayingQueue = false;
|
||||
if (this.ttsSynth) this.ttsSynth.cancel();
|
||||
if (this.currentAudioElement) {
|
||||
this.currentAudioElement.pause();
|
||||
this.currentAudioElement = null;
|
||||
}
|
||||
if (this.activeReject) {
|
||||
this.activeReject(new Error("playback_stopped"));
|
||||
this.activeReject = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Arrêter immédiatement de parler et vider la file
|
||||
public stopSpeaking() {
|
||||
this.audioQueue = [];
|
||||
this.isPlayingQueue = false;
|
||||
if (this.ttsSynth) {
|
||||
this.ttsSynth.cancel();
|
||||
}
|
||||
if (this.currentAudioElement) {
|
||||
this.currentAudioElement.pause();
|
||||
this.currentAudioElement = null;
|
||||
}
|
||||
if (this.activeReject) {
|
||||
this.activeReject(new Error("playback_stopped"));
|
||||
this.activeReject = null;
|
||||
}
|
||||
}
|
||||
|
||||
// Démarrer l'écoute vocale API via AudioContext (WAV Recorder)
|
||||
private async startListeningAPI(
|
||||
onResult: (text: string) => void,
|
||||
onStart: () => void,
|
||||
onEnd: () => void,
|
||||
onError: (err: string) => void,
|
||||
lang: string
|
||||
) {
|
||||
if (this.isListeningActive) {
|
||||
this.stopListening();
|
||||
}
|
||||
|
||||
// Assigner les callbacks pour y avoir accès à l'arrêt
|
||||
this.activeOnResult = onResult;
|
||||
this.activeOnEnd = onEnd;
|
||||
this.activeOnError = onError;
|
||||
this.activeLang = lang;
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
this.micStream = stream;
|
||||
this.leftChannel = [];
|
||||
this.recordingLength = 0;
|
||||
|
||||
const AudioCtx = window.AudioContext || (window as any).webkitAudioContext;
|
||||
this.audioContext = new AudioCtx();
|
||||
|
||||
// Assurer que l'AudioContext est actif (non suspendu)
|
||||
if (this.audioContext.state === 'suspended') {
|
||||
this.logWarn("STT", "AudioContext initialisé à l'état suspendu. Tentative de reprise...");
|
||||
await this.audioContext.resume();
|
||||
}
|
||||
|
||||
this.sampleRate = this.audioContext.sampleRate;
|
||||
this.logInfo("STT", "AudioContext démarré avec succès", {
|
||||
state: this.audioContext.state,
|
||||
sampleRate: this.sampleRate
|
||||
});
|
||||
|
||||
const source = this.audioContext.createMediaStreamSource(stream);
|
||||
// bufferSize de 2048, 1 canal d'entrée, 1 de sortie
|
||||
this.scriptProcessor = this.audioContext.createScriptProcessor(2048, 1, 1);
|
||||
|
||||
this.scriptProcessor.onaudioprocess = (e) => {
|
||||
if (!this.isListeningActive) return;
|
||||
const inputData = e.inputBuffer.getChannelData(0);
|
||||
// Cloner les échantillons
|
||||
this.leftChannel.push(new Float32Array(inputData));
|
||||
this.recordingLength += inputData.length;
|
||||
};
|
||||
|
||||
source.connect(this.scriptProcessor);
|
||||
this.scriptProcessor.connect(this.audioContext.destination);
|
||||
|
||||
this.isListeningActive = true;
|
||||
onStart();
|
||||
} catch (err: any) {
|
||||
this.isListeningActive = false;
|
||||
this.logError("STT", "Impossible d'accéder au microphone", err);
|
||||
onError(err.message || "Impossible d'accéder au microphone.");
|
||||
}
|
||||
}
|
||||
|
||||
// Arrêter proprement la capture AudioContext et exporter en WAV
|
||||
private stopListeningAPI() {
|
||||
if (!this.isListeningActive) return;
|
||||
this.isListeningActive = false;
|
||||
|
||||
// Déconnecter le processeur
|
||||
if (this.scriptProcessor) {
|
||||
this.scriptProcessor.disconnect();
|
||||
this.scriptProcessor.onaudioprocess = null;
|
||||
this.scriptProcessor = null;
|
||||
}
|
||||
|
||||
// Arrêter le flux micro
|
||||
if (this.micStream) {
|
||||
this.micStream.getTracks().forEach(track => track.stop());
|
||||
this.micStream = null;
|
||||
}
|
||||
|
||||
// Fermer l'audio context
|
||||
if (this.audioContext) {
|
||||
this.audioContext.close();
|
||||
this.audioContext = null;
|
||||
}
|
||||
|
||||
const onEndCallback = this.activeOnEnd;
|
||||
const onResultCallback = this.activeOnResult;
|
||||
const onErrorCallback = this.activeOnError;
|
||||
const lang = this.activeLang;
|
||||
|
||||
if (onEndCallback) onEndCallback();
|
||||
|
||||
if (this.recordingLength === 0) {
|
||||
if (onErrorCallback) onErrorCallback("Aucune donnée audio capturée");
|
||||
return;
|
||||
}
|
||||
|
||||
// Exporter le tampon audio en fichier WAV
|
||||
const wavBlob = this.exportWAV(this.leftChannel, this.recordingLength, this.sampleRate);
|
||||
|
||||
this.logInfo("STT", "Arrêt de l'écoute, exportation WAV...", {
|
||||
recordingLength: this.recordingLength,
|
||||
sampleRate: this.sampleRate,
|
||||
wavSize: wavBlob.size
|
||||
});
|
||||
|
||||
this.transcribeAudioWithAPI(wavBlob, lang)
|
||||
.then(text => {
|
||||
if (onResultCallback) onResultCallback(text);
|
||||
})
|
||||
.catch(err => {
|
||||
this.logError("STT", "Erreur de transcription", err);
|
||||
if (onErrorCallback) onErrorCallback(err.message || "Erreur de transcription de l'API");
|
||||
});
|
||||
}
|
||||
|
||||
// Fonctions utilitaires pour générer le conteneur WAV PCM 16-bit
|
||||
private exportWAV(channelData: Float32Array[], recordingLength: number, sampleRate: number): Blob {
|
||||
const originalBuffer = this.mergeBuffers(channelData, recordingLength);
|
||||
|
||||
// Rééchantillonner à 16000 Hz pour la compatibilité avec l'API ASR (Whisper / Qwen3-ASR)
|
||||
const targetSampleRate = 16000;
|
||||
const resampledBuffer = this.downsampleBuffer(originalBuffer, sampleRate, targetSampleRate);
|
||||
|
||||
const wavBuffer = new ArrayBuffer(44 + resampledBuffer.length * 2);
|
||||
const view = new DataView(wavBuffer);
|
||||
|
||||
/* Identifiant RIFF */
|
||||
this.writeString(view, 0, 'RIFF');
|
||||
/* Taille du fichier */
|
||||
view.setUint32(4, 36 + resampledBuffer.length * 2, true);
|
||||
/* Type RIFF */
|
||||
this.writeString(view, 8, 'WAVE');
|
||||
/* Identifiant format fmt */
|
||||
this.writeString(view, 12, 'fmt ');
|
||||
/* Longueur du chunk format */
|
||||
view.setUint32(16, 16, true);
|
||||
/* Format d'encodage (1 = PCM non compressé) */
|
||||
view.setUint16(20, 1, true);
|
||||
/* Nombre de canaux (1 = Mono) */
|
||||
view.setUint16(22, 1, true);
|
||||
/* Fréquence d'échantillonnage */
|
||||
view.setUint32(24, targetSampleRate, true);
|
||||
/* Débit d'octets (targetSampleRate * blockAlign) */
|
||||
view.setUint32(28, targetSampleRate * 2, true);
|
||||
/* Block align (1 canal * 2 octets par échantillon) */
|
||||
view.setUint16(32, 2, true);
|
||||
/* Bits par échantillon (16 bits) */
|
||||
view.setUint16(34, 16, true);
|
||||
/* Identifiant de données data */
|
||||
this.writeString(view, 36, 'data');
|
||||
/* Longueur des données audio */
|
||||
view.setUint32(40, resampledBuffer.length * 2, true);
|
||||
|
||||
this.floatTo16BitPCM(view, 44, resampledBuffer);
|
||||
|
||||
return new Blob([view], { type: 'audio/wav' });
|
||||
}
|
||||
|
||||
private downsampleBuffer(buffer: Float32Array, originalSampleRate: number, targetSampleRate: number): Float32Array {
|
||||
if (originalSampleRate === targetSampleRate) {
|
||||
return buffer;
|
||||
}
|
||||
|
||||
const sampleRateRatio = originalSampleRate / targetSampleRate;
|
||||
const newLength = Math.round(buffer.length / sampleRateRatio);
|
||||
const result = new Float32Array(newLength);
|
||||
|
||||
let offsetResult = 0;
|
||||
let offsetBuffer = 0;
|
||||
|
||||
while (offsetResult < result.length) {
|
||||
const nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
|
||||
|
||||
// Moyennage simple (box filter) pour éviter l'aliasing
|
||||
let accum = 0;
|
||||
let count = 0;
|
||||
for (let i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
|
||||
accum += buffer[i];
|
||||
count++;
|
||||
}
|
||||
|
||||
if (count > 0) {
|
||||
result[offsetResult] = accum / count;
|
||||
} else {
|
||||
result[offsetResult] = buffer[Math.min(offsetBuffer, buffer.length - 1)];
|
||||
}
|
||||
|
||||
offsetResult++;
|
||||
offsetBuffer = nextOffsetBuffer;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private mergeBuffers(channelData: Float32Array[], recordingLength: number): Float32Array {
|
||||
const result = new Float32Array(recordingLength);
|
||||
let offset = 0;
|
||||
for (let i = 0; i < channelData.length; i++) {
|
||||
result.set(channelData[i], offset);
|
||||
offset += channelData[i].length;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private writeString(view: DataView, offset: number, string: string) {
|
||||
for (let i = 0; i < string.length; i++) {
|
||||
view.setUint8(offset + i, string.charCodeAt(i));
|
||||
}
|
||||
}
|
||||
|
||||
private floatTo16BitPCM(output: DataView, offset: number, input: Float32Array) {
|
||||
for (let i = 0; i < input.length; i++, offset += 2) {
|
||||
const s = Math.max(-1, Math.min(1, input[i]));
|
||||
output.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
|
||||
}
|
||||
}
|
||||
|
||||
// Requête API Speech-To-Text compatible OpenAI /v1/audio/transcriptions
|
||||
private async transcribeAudioWithAPI(audioBlob: Blob, lang: string): Promise<string> {
|
||||
if (!this.sttApiUrl) {
|
||||
throw new Error("L'URL de l'API STT n'est pas configurée.");
|
||||
}
|
||||
|
||||
const base = this.sttApiUrl.replace(/\/$/, '');
|
||||
let endpoint: string;
|
||||
if (base.endsWith('/transcriptions')) {
|
||||
endpoint = base;
|
||||
} else if (base.endsWith('/v1')) {
|
||||
endpoint = `${base}/audio/transcriptions`;
|
||||
} else {
|
||||
endpoint = `${base}/v1/audio/transcriptions`;
|
||||
}
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', audioBlob, 'audio.wav');
|
||||
formData.append('model', this.sttModel || 'Qwen/Qwen3-ASR-0.6B');
|
||||
|
||||
if (lang) {
|
||||
const isoLang = lang.split('-')[0];
|
||||
formData.append('language', isoLang);
|
||||
}
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.sttApiKey) {
|
||||
headers['Authorization'] = `Bearer ${this.sttApiKey.trim()}`;
|
||||
}
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: formData
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const rawErr = await response.text().catch(() => '(no body)');
|
||||
let errMsg = `Erreur STT API: ${response.status}`;
|
||||
try {
|
||||
const parsed = JSON.parse(rawErr);
|
||||
errMsg = parsed.error?.message || parsed.message || rawErr;
|
||||
} catch {}
|
||||
throw new Error(errMsg);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
if (data && typeof data.text === 'string') {
|
||||
return data.text;
|
||||
}
|
||||
|
||||
throw new Error("Format de réponse de l'API STT invalide.");
|
||||
}
|
||||
|
||||
// Lancer l'écoute vocale (STT - Speech-To-Text)
|
||||
public startListening(
|
||||
onResult: (text: string) => void,
|
||||
onStart: () => void,
|
||||
onEnd: () => void,
|
||||
onError: (err: string) => void,
|
||||
lang: string = 'fr-FR'
|
||||
) {
|
||||
if (this.sttProvider === 'qwen3-asr') {
|
||||
this.startListeningAPI(onResult, onStart, onEnd, onError, lang);
|
||||
} else {
|
||||
if (!this.recognition) {
|
||||
onError("STT non supporté par ce navigateur");
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.isListeningActive) {
|
||||
this.recognition.stop();
|
||||
}
|
||||
|
||||
this.recognition.lang = lang;
|
||||
this.isListeningActive = true;
|
||||
|
||||
this.recognition.onstart = () => {
|
||||
onStart();
|
||||
};
|
||||
|
||||
this.recognition.onresult = (event: any) => {
|
||||
const resultText = event.results[0][0].transcript;
|
||||
onResult(resultText);
|
||||
};
|
||||
|
||||
this.recognition.onerror = (event: any) => {
|
||||
onError(event.error);
|
||||
};
|
||||
|
||||
this.recognition.onend = () => {
|
||||
this.isListeningActive = false;
|
||||
onEnd();
|
||||
};
|
||||
|
||||
try {
|
||||
this.recognition.start();
|
||||
} catch (e: any) {
|
||||
onError(e.message || "Erreur de démarrage de l'écoute");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Arrêter l'écoute vocale
|
||||
public stopListening() {
|
||||
if (this.sttProvider === 'qwen3-asr') {
|
||||
this.stopListeningAPI();
|
||||
} else {
|
||||
if (this.recognition && this.isListeningActive) {
|
||||
this.recognition.stop();
|
||||
this.isListeningActive = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
/**
|
||||
* Conversation orchestrator: sends user input to Hermes through the resolved transport,
|
||||
* streams the answer into the chat store, drives the HUD state, speaks the reply and
|
||||
* surfaces approvals / clarifications as pending requests.
|
||||
*/
|
||||
import type { HermesPushEvent } from '../../shared/ipc';
|
||||
import { Log } from '../lib/log';
|
||||
import { previewText } from '../lib/text';
|
||||
import { useChat, type PendingRequest } from '../state/chat';
|
||||
import { useHermes } from '../state/hermes';
|
||||
import { useSettings } from '../state/settings';
|
||||
import { executeLocalTool, LOCAL_TOOL_DEFINITIONS } from './hermes/localTools';
|
||||
import type { HermesStreamEvent, SendHandle } from './hermes/types';
|
||||
import { speech } from './voice/speech';
|
||||
|
||||
let active: SendHandle | null = null;
|
||||
let activeMessageId: string | null = null;
|
||||
|
||||
const toolLabel = (name: string) => name.replace(/_/g, ' ');
|
||||
|
||||
function localToolContext() {
|
||||
const chat = useChat.getState();
|
||||
const settings = useSettings.getState().settings;
|
||||
return {
|
||||
setEmotion: (state: string) => {
|
||||
const map: Record<string, Parameters<typeof chat.setHudOverride>[0]> = {
|
||||
happy: 'success', success: 'success', thinking: 'thinking', alert: 'alert', error: 'error', neutral: null
|
||||
};
|
||||
chat.setHudOverride(map[state] ?? null);
|
||||
setTimeout(() => useChat.getState().setHudOverride(null), 6000);
|
||||
},
|
||||
speak: (text: string) => speech.say(text),
|
||||
getStatus: () => ({
|
||||
transport: useHermes.getState().transport,
|
||||
link: useHermes.getState().link,
|
||||
assistant: settings.assistantName,
|
||||
messages: chat.messages.length,
|
||||
speaking: speech.isSpeaking(),
|
||||
handsFree: settings.voice.handsFree
|
||||
}),
|
||||
getHistory: (n: number) => chat.messages.slice(-n).map((m) => ({ role: m.role, content: m.content })),
|
||||
notify: (title: string, body: string) => {
|
||||
try {
|
||||
if (Notification.permission === 'granted') new Notification(title, { body });
|
||||
} catch {
|
||||
/* notifications unavailable */
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function handleEvent(event: HermesStreamEvent, messageId: string, runIdRef: { id: string | null }): void {
|
||||
const chat = useChat.getState();
|
||||
switch (event.kind) {
|
||||
case 'run.started':
|
||||
runIdRef.id = event.runId;
|
||||
chat.setSending(true, event.runId);
|
||||
break;
|
||||
case 'delta':
|
||||
chat.appendToMessage(messageId, event.text);
|
||||
speech.pushStream(event.text);
|
||||
break;
|
||||
case 'reasoning':
|
||||
chat.appendToMessage(messageId, event.text, 'reasoning');
|
||||
break;
|
||||
case 'tool.start':
|
||||
chat.pushActivity({ id: event.id, kind: 'tool', name: event.name, status: 'running', detail: event.args ? previewText(event.args, 160) : undefined });
|
||||
chat.setHud('thinking');
|
||||
break;
|
||||
case 'tool.progress':
|
||||
if (!chat.activity.some((a) => (event.id && a.id === event.id) || (a.name === event.name && a.status === 'running'))) {
|
||||
chat.pushActivity({ id: event.id, kind: 'tool', name: event.name, status: 'running', detail: event.label ?? event.detail });
|
||||
}
|
||||
break;
|
||||
case 'tool.end':
|
||||
chat.finishActivity({ id: event.id || undefined, name: event.name }, {
|
||||
status: event.ok ? 'done' : 'error',
|
||||
output: event.output ? previewText(event.output, 400) : undefined
|
||||
});
|
||||
break;
|
||||
case 'subagent.start':
|
||||
chat.pushActivity({ id: event.id, kind: 'subagent', name: event.label ?? 'sous-agent', status: 'running', detail: event.childSessionId });
|
||||
break;
|
||||
case 'subagent.end':
|
||||
chat.finishActivity({ id: event.id }, { status: event.status.includes('fail') ? 'error' : 'done', output: event.summary });
|
||||
break;
|
||||
case 'approval.request':
|
||||
addPending({
|
||||
requestId: event.requestId,
|
||||
runId: runIdRef.id ?? '',
|
||||
kind: 'approval',
|
||||
title: 'Autorisation requise',
|
||||
description: event.description,
|
||||
options: event.options,
|
||||
tool: event.tool,
|
||||
args: event.args
|
||||
});
|
||||
break;
|
||||
case 'clarify.request':
|
||||
addPending({ requestId: event.requestId, runId: runIdRef.id ?? '', kind: 'clarify', title: 'Précision demandée', description: event.question, options: event.options });
|
||||
break;
|
||||
case 'input.request':
|
||||
addPending({ requestId: event.requestId, runId: runIdRef.id ?? '', kind: 'input', title: event.variant === 'sudo' ? 'Mot de passe sudo' : 'Secret requis', description: event.prompt, secret: true });
|
||||
break;
|
||||
case 'request.expire':
|
||||
chat.removePending(event.requestId);
|
||||
break;
|
||||
case 'session':
|
||||
if (event.sessionId && event.sessionId !== useSettings.getState().settings.hermesSessionId) {
|
||||
useSettings.getState().setHermesSessionId(event.sessionId);
|
||||
Log.info('hermes', `session id updated: ${event.sessionId}`);
|
||||
}
|
||||
break;
|
||||
case 'completed':
|
||||
if (event.usage) chat.updateMessage(messageId, { usage: event.usage });
|
||||
break;
|
||||
case 'error':
|
||||
chat.setError(event.message);
|
||||
break;
|
||||
case 'raw':
|
||||
Log.debug('hermes', `event ${event.event}`, event.data);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
function addPending(request: Omit<PendingRequest, 'id' | 'createdAt'>): void {
|
||||
const chat = useChat.getState();
|
||||
chat.addPending(request);
|
||||
chat.setHud('alert');
|
||||
speech.say(request.kind === 'approval' ? 'Autorisation requise.' : request.kind === 'clarify' ? request.description : 'Saisie requise.');
|
||||
}
|
||||
|
||||
export async function sendMessage(text: string, images: string[] = [], source = 'eveflow'): Promise<void> {
|
||||
const trimmed = text.trim();
|
||||
if ((!trimmed && images.length === 0) || active) return;
|
||||
const chat = useChat.getState();
|
||||
const hermes = useHermes.getState();
|
||||
const settings = useSettings.getState().settings;
|
||||
|
||||
speech.stop();
|
||||
chat.setError(null);
|
||||
chat.addMessage({ role: 'user', content: trimmed || '(image)', images: images.length ? images : undefined, source, status: 'done' });
|
||||
const history = chat.messages
|
||||
.filter((m) => m.role !== 'system' && m.status !== 'error')
|
||||
.slice(-12)
|
||||
.map((m) => ({ role: m.role as 'user' | 'assistant', content: m.content }));
|
||||
const messageId = chat.addMessage({ role: 'assistant', content: '', status: 'streaming', transport: hermes.transport });
|
||||
activeMessageId = messageId;
|
||||
chat.setSending(true);
|
||||
chat.setHud('thinking');
|
||||
const startedAt = Date.now();
|
||||
const runIdRef = { id: null as string | null };
|
||||
|
||||
const client = hermes.client();
|
||||
const handle = client.send(
|
||||
{
|
||||
text: trimmed || 'Analyse cette image.',
|
||||
images,
|
||||
history,
|
||||
sessionId: settings.hermesSessionId,
|
||||
onEvent: (event) => handleEvent(event, messageId, runIdRef),
|
||||
localToolDefinitions: LOCAL_TOOL_DEFINITIONS,
|
||||
localToolExecutor: (name, args) => executeLocalTool(name, args, localToolContext())
|
||||
},
|
||||
hermes.transport
|
||||
);
|
||||
active = handle;
|
||||
chat.updateMessage(messageId, { transport: handle.transport });
|
||||
|
||||
try {
|
||||
const reply = await handle.result;
|
||||
const current = useChat.getState().messages.find((m) => m.id === messageId);
|
||||
const finalText = reply || current?.content || '';
|
||||
useChat.getState().updateMessage(messageId, { content: finalText, status: 'done' });
|
||||
// Anything not streamed (non-delta transports) is spoken now.
|
||||
if (settings.speech.autoSpeak) {
|
||||
if (!current?.content && finalText) speech.say(finalText);
|
||||
else speech.endStream();
|
||||
} else {
|
||||
speech.discardStream();
|
||||
}
|
||||
useChat.getState().setHud(speech.isSpeaking() ? 'speaking' : 'idle');
|
||||
if (useHermes.getState().link === 'offline') useHermes.getState().connect().catch(() => undefined);
|
||||
Log.info('hermes', `reply in ${Date.now() - startedAt} ms via ${handle.transport}`, { chars: finalText.length });
|
||||
} catch (err) {
|
||||
const message = (err as Error).message || String(err);
|
||||
const partial = useChat.getState().messages.find((m) => m.id === messageId)?.content ?? '';
|
||||
useChat.getState().updateMessage(messageId, {
|
||||
content: partial || `Liaison Hermes interrompue : ${message}`,
|
||||
status: partial ? 'cancelled' : 'error'
|
||||
});
|
||||
useChat.getState().setError(message);
|
||||
useChat.getState().setHud('error');
|
||||
speech.discardStream();
|
||||
Log.error('hermes', `send failed: ${message}`);
|
||||
if (/HTTP|ECONNREFUSED|fetch|réseau|network/i.test(message)) useHermes.getState().connect().catch(() => undefined);
|
||||
setTimeout(() => {
|
||||
if (useChat.getState().hud === 'error') useChat.getState().setHud('idle');
|
||||
}, 4000);
|
||||
} finally {
|
||||
active = null;
|
||||
activeMessageId = null;
|
||||
useChat.getState().setSending(false);
|
||||
for (const a of useChat.getState().activity) {
|
||||
if (a.status === 'running') useChat.getState().finishActivity({ id: a.id }, { status: 'done' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function stopGeneration(): void {
|
||||
if (active) {
|
||||
active.abort();
|
||||
Log.info('hermes', 'generation aborted by user');
|
||||
}
|
||||
speech.stop();
|
||||
if (activeMessageId) useChat.getState().updateMessage(activeMessageId, { status: 'cancelled' });
|
||||
useChat.getState().setHud('idle');
|
||||
}
|
||||
|
||||
export async function steer(text: string): Promise<boolean> {
|
||||
const runId = useChat.getState().currentRunId;
|
||||
if (!runId || !text.trim()) return false;
|
||||
try {
|
||||
await useHermes.getState().client().steerRun(runId, text.trim());
|
||||
useChat.getState().addMessage({ role: 'system', content: `Consigne transmise à Hermes : ${text.trim()}`, status: 'done' });
|
||||
return true;
|
||||
} catch (err) {
|
||||
useChat.getState().setError(`Steer impossible : ${(err as Error).message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolvePending(request: PendingRequest, decision: string): Promise<void> {
|
||||
const chat = useChat.getState();
|
||||
chat.removePending(request.id);
|
||||
const client = useHermes.getState().client();
|
||||
try {
|
||||
if (request.kind === 'approval') {
|
||||
const choice = (['once', 'session', 'always', 'deny'].includes(decision) ? decision : decision === 'approve' ? 'once' : 'deny') as 'once' | 'session' | 'always' | 'deny';
|
||||
if (!request.runId) throw new Error('run inconnu');
|
||||
await client.approveRun(request.runId, choice, request.requestId || undefined);
|
||||
chat.addMessage({ role: 'system', content: `Autorisation ${choice === 'deny' ? 'refusée' : 'accordée'}${request.tool ? ` (${toolLabel(request.tool)})` : ''}.`, status: 'done' });
|
||||
} else {
|
||||
// Clarifications and secrets are answered by steering the run with the reply.
|
||||
if (!request.runId) throw new Error('run inconnu');
|
||||
await client.steerRun(request.runId, decision);
|
||||
chat.addMessage({ role: 'system', content: request.secret ? 'Saisie transmise à Hermes.' : `Réponse transmise : ${decision}`, status: 'done' });
|
||||
}
|
||||
chat.setHud(useChat.getState().isSending ? 'thinking' : 'idle');
|
||||
} catch (err) {
|
||||
chat.setError(`Réponse impossible : ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Incoming webhook pushes (Telegram mirror, cron deliveries…). */
|
||||
export function handlePush(event: HermesPushEvent): void {
|
||||
const chat = useChat.getState();
|
||||
const settings = useSettings.getState().settings;
|
||||
if (event.type === 'raw') {
|
||||
Log.debug('webhook', 'raw push', event.raw);
|
||||
chat.pushActivity({ kind: 'system', name: `webhook ${event.source}`, status: 'done', detail: previewText(event.text, 200) });
|
||||
return;
|
||||
}
|
||||
if (!event.text.trim() && !event.images?.length) return;
|
||||
chat.addMessage({
|
||||
role: event.role === 'user' ? 'user' : 'assistant',
|
||||
content: event.text,
|
||||
images: event.images,
|
||||
source: event.source,
|
||||
jobName: event.jobName,
|
||||
status: 'done'
|
||||
});
|
||||
if (event.type === 'job') {
|
||||
chat.pushActivity({ kind: 'job', name: event.jobName || 'cron', status: event.status?.includes('fail') ? 'error' : 'done', detail: previewText(event.text, 160) });
|
||||
void useHermes.getState().refreshJobs();
|
||||
}
|
||||
if (event.role !== 'user' && settings.speech.speakIncoming) {
|
||||
speech.say(event.jobName ? `Résultat de ${event.jobName}. ${event.text}` : event.text);
|
||||
}
|
||||
chat.setHud(event.status?.includes('fail') ? 'alert' : 'success');
|
||||
setTimeout(() => {
|
||||
const s = useChat.getState();
|
||||
if (s.hud === 'success' || s.hud === 'alert') s.setHud(speech.isSpeaking() ? 'speaking' : 'idle');
|
||||
}, 3000);
|
||||
}
|
||||
@@ -1,163 +0,0 @@
|
||||
export type EveEmotion = 'neutral' | 'happy' | 'sad' | 'surprised' | 'thinking' | 'angry';
|
||||
export type EveAvatar = 'eve' | 'nova' | 'aegis';
|
||||
|
||||
export class EmotionService {
|
||||
// Analyser le sentiment d'une phrase pour en déduire l'émotion d'Eve
|
||||
public static analyzeSentiment(text: string): EveEmotion {
|
||||
const cleanText = text.toLowerCase().trim();
|
||||
|
||||
// Mots-clés pour la joie / l'excitation
|
||||
const happyKeywords = [
|
||||
'génial', 'super', 'ravi', 'heureux', 'joie', 'plaisir', 'gagné', 'haha', 'cool', 'géniale',
|
||||
'merci', 'adore', 'excellent', 'parfait', 'extraordinaire', 'bravo', 'sourire', 'heureuse',
|
||||
'magnifique', 'awesome', 'great', 'happy', 'love', 'smile'
|
||||
];
|
||||
|
||||
// Mots-clés pour la tristesse / la déception
|
||||
const sadKeywords = [
|
||||
'désolé', 'triste', 'dommage', 'regret', 'malheureusement', 'pleurer', 'hélas', 'malheur',
|
||||
'perdu', 'sombre', 'erreur', 'échec', 'souffrir', 'bad', 'sorry', 'sad', 'unfortunately'
|
||||
];
|
||||
|
||||
// Mots-clés pour la colère / le mécontentement
|
||||
const angryKeywords = [
|
||||
'non', 'colère', 'énervé', 'fâché', 'nul', 'inacceptable', 'mauvais', 'déteste', 'pire',
|
||||
'irritant', 'frustré', 'stupide', 'ridicule', 'assez', 'stop', 'angry', 'no', 'hate'
|
||||
];
|
||||
|
||||
// Mots-clés pour la surprise / l'émerveillement
|
||||
const surprisedKeywords = [
|
||||
'wow', 'quoi', 'incroyable', 'vraiment', 'surprise', 'surpris', 'soudain', 'extraordinaire',
|
||||
'inattendu', 'magique', 'oh', 'ah', 'bizarre', 'étrange', 'wow', 'incredible', 'surprise'
|
||||
];
|
||||
|
||||
// Mots-clés pour la réflexion / l'analyse
|
||||
const thinkingKeywords = [
|
||||
'réfléchir', 'analyse', 'calcul', 'comprendre', 'hmm', 'euh', 'peut-être', 'probablement',
|
||||
'possible', 'hypothèse', 'cherche', 'étudie', 'question', 'pourquoi', 'comment', 'pense',
|
||||
'thinking', 'ponder', 'analyze', 'search', 'perhaps', 'maybe'
|
||||
];
|
||||
|
||||
// 1. Analyse par présence de mots-clés prioritaires
|
||||
if (this.containsAny(cleanText, angryKeywords)) return 'angry';
|
||||
if (this.containsAny(cleanText, sadKeywords)) return 'sad';
|
||||
if (this.containsAny(cleanText, surprisedKeywords)) return 'surprised';
|
||||
if (this.containsAny(cleanText, happyKeywords)) return 'happy';
|
||||
if (this.containsAny(cleanText, thinkingKeywords)) return 'thinking';
|
||||
|
||||
// 2. Analyse par ponctuation si aucun mot-clé n'est saillant
|
||||
if (cleanText.includes('!!!') || cleanText.includes('wow')) return 'surprised';
|
||||
if (cleanText.includes('?') && cleanText.length < 50) return 'thinking'; // Petite question rapide
|
||||
if (cleanText.includes('...')) return 'thinking';
|
||||
|
||||
// Émotion par défaut
|
||||
return 'neutral';
|
||||
}
|
||||
|
||||
// Vérifier si le texte contient au moins un des mots-clés
|
||||
private static containsAny(text: string, keywords: string[]): boolean {
|
||||
return keywords.some(keyword => {
|
||||
// Expression régulière pour rechercher le mot exact (évite les correspondances partielles fausses)
|
||||
const regex = new RegExp(`\\b${keyword}\\b|${keyword}`, 'i');
|
||||
return regex.test(text);
|
||||
});
|
||||
}
|
||||
|
||||
// Renvoie les paramètres graphiques (couleur des yeux, intensité, forme) pour chaque émotion et avatar
|
||||
// Utile pour la 3D ou les shaders
|
||||
public static getEmotionStyle(emotion: EveEmotion, avatarId: EveAvatar = 'eve') {
|
||||
// Spectre de couleurs par défaut selon l'avatar sélectionné
|
||||
let eyeColor = '#00f2ff'; // Eve par défaut
|
||||
|
||||
if (avatarId === 'nova') {
|
||||
switch (emotion) {
|
||||
case 'happy': eyeColor = '#ff0077'; break; // Luminous hot pink
|
||||
case 'sad': eyeColor = '#ff88dd'; break; // Melancholic soft pink
|
||||
case 'angry': eyeColor = '#9900ff'; break; // Furious deep purple
|
||||
case 'surprised': eyeColor = '#ffc8ff'; break; // Electric light pink
|
||||
case 'thinking': eyeColor = '#c300ff'; break; // Luminous neon violet
|
||||
case 'neutral':
|
||||
default:
|
||||
eyeColor = '#ff00ea'; // Vibrant magenta
|
||||
break;
|
||||
}
|
||||
} else if (avatarId === 'aegis') {
|
||||
switch (emotion) {
|
||||
case 'happy': eyeColor = '#ffaa00'; break; // Luminous warm gold
|
||||
case 'sad': eyeColor = '#ffa677'; break; // Melancholic dusty orange
|
||||
case 'angry': eyeColor = '#ff3300'; break; // Furious Nixie warning red
|
||||
case 'surprised': eyeColor = '#ffe4cc'; break; // Electric light amber
|
||||
case 'thinking': eyeColor = '#ff8800'; break; // Luminous bright orange
|
||||
case 'neutral':
|
||||
default:
|
||||
eyeColor = '#ff6a00'; // Vintage Nixie amber
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
// 'eve' original (Cyan spectrum)
|
||||
switch (emotion) {
|
||||
case 'happy': eyeColor = '#00ffff'; break; // Luminous turquoise cyan
|
||||
case 'sad': eyeColor = '#3de7ff'; break; // Melancholic luminous soft cyan
|
||||
case 'angry': eyeColor = '#00a2ff'; break; // Furious electric cyan-blue
|
||||
case 'surprised': eyeColor = '#99f2ff'; break; // Electric light ice blue
|
||||
case 'thinking': eyeColor = '#00c3ff'; break; // Luminous sky blue
|
||||
case 'neutral':
|
||||
default:
|
||||
eyeColor = '#00f2ff'; // Signature vibrant cyan blue
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
switch (emotion) {
|
||||
case 'happy':
|
||||
return {
|
||||
eyeColor,
|
||||
eyeScaleY: 0.6,
|
||||
eyeRotation: 0.1, // Joyous rotation
|
||||
pulseSpeed: 3.5,
|
||||
lightIntensity: 2.0
|
||||
};
|
||||
case 'sad':
|
||||
return {
|
||||
eyeColor,
|
||||
eyeScaleY: 0.45,
|
||||
eyeRotation: -0.15, // Drooping eyes
|
||||
pulseSpeed: 1.0,
|
||||
lightIntensity: 1.0
|
||||
};
|
||||
case 'angry':
|
||||
return {
|
||||
eyeColor,
|
||||
eyeScaleY: 0.3,
|
||||
eyeRotation: 0.25, // Angry angled eyes
|
||||
pulseSpeed: 4.0,
|
||||
lightIntensity: 1.8
|
||||
};
|
||||
case 'surprised':
|
||||
return {
|
||||
eyeColor,
|
||||
eyeScaleY: 1.0, // Fully open round eyes
|
||||
eyeRotation: 0,
|
||||
pulseSpeed: 5.0,
|
||||
lightIntensity: 2.5
|
||||
};
|
||||
case 'thinking':
|
||||
return {
|
||||
eyeColor,
|
||||
eyeScaleY: 0.5, // Half closed thinking eyes
|
||||
eyeRotation: -0.05,
|
||||
pulseSpeed: 1.8,
|
||||
lightIntensity: 1.3
|
||||
};
|
||||
case 'neutral':
|
||||
default:
|
||||
return {
|
||||
eyeColor,
|
||||
eyeScaleY: 0.65, // Standard oblong shape
|
||||
eyeRotation: 0,
|
||||
pulseSpeed: 2.0,
|
||||
lightIntensity: 1.5
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,492 @@
|
||||
/**
|
||||
* Full Hermes Agent API client: discovery, runs (SSE lifecycle + approvals + steering),
|
||||
* sessions (server-side memory), OpenAI-compatible chat completions (with EveFlow tools)
|
||||
* and the cron jobs REST API.
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { SseParser, tryParseJson } from '../../lib/sse';
|
||||
import { httpFetch, httpStream, HttpError, type StreamHandle } from '../../lib/transport';
|
||||
import { normalizeCompletionChunk, normalizeLifecycleEvent, type CompletionToolCallAccumulator } from './events';
|
||||
import type {
|
||||
HermesCapabilities,
|
||||
HermesChatMessage,
|
||||
HermesConfig,
|
||||
HermesHealth,
|
||||
HermesJob,
|
||||
HermesJobDraft,
|
||||
HermesModel,
|
||||
HermesRunInfo,
|
||||
HermesSession,
|
||||
HermesSessionMessage,
|
||||
HermesSkill,
|
||||
HermesStreamEvent,
|
||||
HermesToolset,
|
||||
HermesTransport,
|
||||
SendHandle,
|
||||
SendOptions
|
||||
} from './types';
|
||||
|
||||
type Rec = Record<string, unknown>;
|
||||
const isRec = (v: unknown): v is Rec => !!v && typeof v === 'object' && !Array.isArray(v);
|
||||
|
||||
export type ResolvedTransport = Exclude<HermesTransport, 'auto'>;
|
||||
|
||||
export function hermesBaseUrl(url: string): string {
|
||||
let base = url.trim().replace(/\/+$/, '');
|
||||
base = base.replace(/\/chat\/completions$/, '');
|
||||
base = base.replace(/\/v1$/, '');
|
||||
return base;
|
||||
}
|
||||
|
||||
function extractArray<T>(payload: unknown, keys: string[]): T[] {
|
||||
if (Array.isArray(payload)) return payload as T[];
|
||||
if (isRec(payload)) {
|
||||
for (const key of keys) {
|
||||
const value = payload[key];
|
||||
if (Array.isArray(value)) return value as T[];
|
||||
}
|
||||
if (isRec(payload.data)) return extractArray<T>(payload.data, keys);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function errorMessage(status: number, body: string): string {
|
||||
const parsed = tryParseJson<Rec>(body);
|
||||
if (parsed) {
|
||||
const err = parsed.error;
|
||||
if (isRec(err) && typeof err.message === 'string') return err.message;
|
||||
if (typeof err === 'string') return err;
|
||||
for (const key of ['message', 'detail', 'error_description']) {
|
||||
if (typeof parsed[key] === 'string') return parsed[key] as string;
|
||||
}
|
||||
}
|
||||
const text = body.trim().slice(0, 240);
|
||||
if (status === 401 || status === 403) return `Authentification refusée (HTTP ${status}). Vérifiez la clé API_SERVER_KEY.`;
|
||||
if (status === 404) return `Endpoint introuvable (HTTP 404)${text ? ` : ${text}` : ''}`;
|
||||
if (status === 429) return 'Hermes est saturé (HTTP 429, max_concurrent_runs atteint).';
|
||||
return `HTTP ${status}${text ? ` : ${text}` : ''}`;
|
||||
}
|
||||
|
||||
export function resolveTransport(config: HermesConfig, caps: HermesCapabilities | null): ResolvedTransport {
|
||||
if (config.transport !== 'auto') return config.transport;
|
||||
if (!caps) return 'completions';
|
||||
const features = caps.features ?? {};
|
||||
const endpoints = caps.endpoints ?? {};
|
||||
if (features.run_submission && features.run_events_sse) return 'runs';
|
||||
if (endpoints.session_chat) return 'sessions';
|
||||
return 'completions';
|
||||
}
|
||||
|
||||
export class HermesClient {
|
||||
constructor(private readonly config: HermesConfig) {}
|
||||
|
||||
get base(): string {
|
||||
return hermesBaseUrl(this.config.url);
|
||||
}
|
||||
|
||||
private headers(extra: Record<string, string> = {}, json = true): Record<string, string> {
|
||||
const h: Record<string, string> = { ...extra };
|
||||
if (json) h['Content-Type'] = 'application/json';
|
||||
h.Accept = h.Accept ?? 'application/json';
|
||||
if (this.config.apiKey.trim()) h.Authorization = `Bearer ${this.config.apiKey.trim()}`;
|
||||
if (this.config.sessionKey.trim()) h['X-Hermes-Session-Key'] = this.config.sessionKey.trim().slice(0, 256);
|
||||
return h;
|
||||
}
|
||||
|
||||
private async request<T>(path: string, init: { method?: 'GET' | 'POST' | 'PATCH' | 'DELETE'; body?: unknown; timeoutMs?: number } = {}): Promise<T> {
|
||||
if (!this.config.url.trim()) throw new Error("URL Hermes non configurée.");
|
||||
const url = `${this.base}${path}`;
|
||||
const res = await httpFetch({
|
||||
url,
|
||||
method: init.method ?? 'GET',
|
||||
headers: this.headers(),
|
||||
body: init.body !== undefined ? JSON.stringify(init.body) : undefined,
|
||||
timeoutMs: init.timeoutMs ?? 20_000
|
||||
});
|
||||
if (!res.ok) throw new HttpError(res.status, errorMessage(res.status, res.text ?? ''), res.text);
|
||||
const text = res.text ?? '';
|
||||
if (!text.trim()) return null as T;
|
||||
const parsed = tryParseJson<T>(text);
|
||||
if (parsed === null) throw new Error(`Réponse Hermes illisible depuis ${path}`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
// ── Discovery ─────────────────────────────────────────────────────────────
|
||||
|
||||
capabilities(): Promise<HermesCapabilities> {
|
||||
return this.request<HermesCapabilities>('/v1/capabilities');
|
||||
}
|
||||
|
||||
async health(): Promise<HermesHealth> {
|
||||
try {
|
||||
return await this.request<HermesHealth>('/health/detailed', { timeoutMs: 8000 });
|
||||
} catch (err) {
|
||||
if (err instanceof HttpError && (err.status === 401 || err.status === 403)) throw err;
|
||||
const basic = await this.request<HermesHealth | Rec>('/health', { timeoutMs: 8000 });
|
||||
return { status: 'ok', ...(isRec(basic) ? basic : {}) } as HermesHealth;
|
||||
}
|
||||
}
|
||||
|
||||
async models(): Promise<HermesModel[]> {
|
||||
const payload = await this.request<unknown>('/v1/models');
|
||||
return extractArray<HermesModel>(payload, ['data', 'models']);
|
||||
}
|
||||
|
||||
async skills(): Promise<HermesSkill[]> {
|
||||
const payload = await this.request<unknown>('/v1/skills');
|
||||
return extractArray<HermesSkill>(payload, ['skills', 'data', 'items']);
|
||||
}
|
||||
|
||||
async toolsets(): Promise<HermesToolset[]> {
|
||||
const payload = await this.request<unknown>('/v1/toolsets');
|
||||
return extractArray<HermesToolset>(payload, ['toolsets', 'data', 'items']);
|
||||
}
|
||||
|
||||
// ── Sessions ──────────────────────────────────────────────────────────────
|
||||
|
||||
async listSessions(limit = 40): Promise<HermesSession[]> {
|
||||
const payload = await this.request<unknown>(`/api/sessions?limit=${limit}`);
|
||||
return extractArray<HermesSession>(payload, ['sessions', 'items', 'data', 'results']);
|
||||
}
|
||||
|
||||
async createSession(title?: string): Promise<HermesSession> {
|
||||
const payload = await this.request<unknown>('/api/sessions', { method: 'POST', body: title ? { title } : {} });
|
||||
if (isRec(payload) && isRec(payload.session)) return payload.session as HermesSession;
|
||||
return payload as HermesSession;
|
||||
}
|
||||
|
||||
getSession(id: string): Promise<HermesSession> {
|
||||
return this.request<HermesSession>(`/api/sessions/${encodeURIComponent(id)}`);
|
||||
}
|
||||
|
||||
async sessionMessages(id: string): Promise<HermesSessionMessage[]> {
|
||||
const payload = await this.request<unknown>(`/api/sessions/${encodeURIComponent(id)}/messages`);
|
||||
return extractArray<HermesSessionMessage>(payload, ['messages', 'items', 'data']);
|
||||
}
|
||||
|
||||
updateSession(id: string, patch: { title?: string; end_reason?: string }): Promise<HermesSession> {
|
||||
return this.request<HermesSession>(`/api/sessions/${encodeURIComponent(id)}`, { method: 'PATCH', body: patch });
|
||||
}
|
||||
|
||||
deleteSession(id: string): Promise<void> {
|
||||
return this.request<void>(`/api/sessions/${encodeURIComponent(id)}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
async forkSession(id: string): Promise<HermesSession> {
|
||||
const payload = await this.request<unknown>(`/api/sessions/${encodeURIComponent(id)}/fork`, { method: 'POST', body: {} });
|
||||
if (isRec(payload) && isRec(payload.session)) return payload.session as HermesSession;
|
||||
return payload as HermesSession;
|
||||
}
|
||||
|
||||
// ── Jobs (cron) ───────────────────────────────────────────────────────────
|
||||
|
||||
async listJobs(): Promise<HermesJob[]> {
|
||||
const payload = await this.request<unknown>('/api/jobs');
|
||||
return extractArray<HermesJob>(payload, ['jobs', 'items', 'data']).map(normalizeJob);
|
||||
}
|
||||
|
||||
private unwrapJob(payload: unknown): HermesJob {
|
||||
if (isRec(payload) && isRec(payload.job)) return normalizeJob(payload.job as HermesJob);
|
||||
return normalizeJob(payload as HermesJob);
|
||||
}
|
||||
|
||||
async createJob(draft: HermesJobDraft): Promise<HermesJob> {
|
||||
const body: Rec = { name: draft.name, schedule: draft.schedule, prompt: draft.prompt };
|
||||
if (draft.deliver) body.deliver = draft.deliver;
|
||||
if (draft.skills?.length) body.skills = draft.skills;
|
||||
return this.unwrapJob(await this.request<unknown>('/api/jobs', { method: 'POST', body }));
|
||||
}
|
||||
|
||||
async updateJob(id: string, patch: Partial<HermesJobDraft>): Promise<HermesJob> {
|
||||
return this.unwrapJob(await this.request<unknown>(`/api/jobs/${encodeURIComponent(id)}`, { method: 'PATCH', body: patch }));
|
||||
}
|
||||
|
||||
deleteJob(id: string): Promise<void> {
|
||||
return this.request<void>(`/api/jobs/${encodeURIComponent(id)}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
async pauseJob(id: string): Promise<HermesJob> {
|
||||
return this.unwrapJob(await this.request<unknown>(`/api/jobs/${encodeURIComponent(id)}/pause`, { method: 'POST', body: {} }));
|
||||
}
|
||||
|
||||
async resumeJob(id: string): Promise<HermesJob> {
|
||||
return this.unwrapJob(await this.request<unknown>(`/api/jobs/${encodeURIComponent(id)}/resume`, { method: 'POST', body: {} }));
|
||||
}
|
||||
|
||||
async runJob(id: string): Promise<HermesJob> {
|
||||
return this.unwrapJob(await this.request<unknown>(`/api/jobs/${encodeURIComponent(id)}/run`, { method: 'POST', body: {}, timeoutMs: 30_000 }));
|
||||
}
|
||||
|
||||
// ── Runs ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async startRun(body: { input: string; session_id?: string; instructions?: string; model?: string }): Promise<{ run_id: string; status: string }> {
|
||||
const payload: Rec = { input: body.input };
|
||||
if (body.session_id) payload.session_id = body.session_id;
|
||||
if (body.instructions) payload.instructions = body.instructions;
|
||||
if (body.model) payload.model = body.model;
|
||||
return this.request<{ run_id: string; status: string }>('/v1/runs', { method: 'POST', body: payload, timeoutMs: 30_000 });
|
||||
}
|
||||
|
||||
getRun(runId: string): Promise<HermesRunInfo> {
|
||||
return this.request<HermesRunInfo>(`/v1/runs/${encodeURIComponent(runId)}`);
|
||||
}
|
||||
|
||||
stopRun(runId: string): Promise<unknown> {
|
||||
return this.request<unknown>(`/v1/runs/${encodeURIComponent(runId)}/stop`, { method: 'POST', body: {} });
|
||||
}
|
||||
|
||||
steerRun(runId: string, text: string): Promise<unknown> {
|
||||
return this.request<unknown>(`/v1/runs/${encodeURIComponent(runId)}/steer`, { method: 'POST', body: { input: text } });
|
||||
}
|
||||
|
||||
approveRun(runId: string, choice: 'once' | 'session' | 'always' | 'deny', requestId?: string): Promise<unknown> {
|
||||
const body: Rec = { choice };
|
||||
if (requestId) body.request_id = requestId;
|
||||
return this.request<unknown>(`/v1/runs/${encodeURIComponent(runId)}/approval`, { method: 'POST', body });
|
||||
}
|
||||
|
||||
/** Subscribe to the lifecycle stream of a run. */
|
||||
async streamRunEvents(runId: string, onEvent: (event: HermesStreamEvent) => void): Promise<StreamHandle> {
|
||||
return this.openSse(`/v1/runs/${encodeURIComponent(runId)}/events`, { method: 'GET' }, onEvent);
|
||||
}
|
||||
|
||||
private async openSse(
|
||||
path: string,
|
||||
init: { method: 'GET' | 'POST'; body?: unknown },
|
||||
onEvent: (event: HermesStreamEvent) => void
|
||||
): Promise<StreamHandle> {
|
||||
const parser = new SseParser((message) => {
|
||||
const data = tryParseJson<unknown>(message.data) ?? message.data;
|
||||
const eventName = message.event !== 'message' ? message.event : isRec(data) && typeof data.event === 'string' ? data.event : 'message';
|
||||
for (const event of normalizeLifecycleEvent(eventName, data)) onEvent(event);
|
||||
});
|
||||
const errorChunks: string[] = [];
|
||||
const handle = await httpStream(
|
||||
{
|
||||
url: `${this.base}${path}`,
|
||||
method: init.method,
|
||||
headers: this.headers({ Accept: 'text/event-stream' }, init.body !== undefined),
|
||||
body: init.body !== undefined ? JSON.stringify(init.body) : undefined,
|
||||
timeoutMs: 15 * 60_000
|
||||
},
|
||||
{ onChunk: (text) => (handleReady ? parser.feed(text) : errorChunks.push(text)) }
|
||||
);
|
||||
let handleReady = handle.start.ok;
|
||||
if (!handle.start.ok) {
|
||||
await handle.done.catch(() => undefined);
|
||||
throw new HttpError(handle.start.status, errorMessage(handle.start.status, errorChunks.join('')));
|
||||
}
|
||||
handleReady = true;
|
||||
const done = handle.done.then(() => parser.end());
|
||||
return { ...handle, done };
|
||||
}
|
||||
|
||||
// ── Unified send ──────────────────────────────────────────────────────────
|
||||
|
||||
send(options: SendOptions, transport: ResolvedTransport): SendHandle {
|
||||
const effective: ResolvedTransport = options.images?.length && transport === 'runs' ? 'completions' : transport;
|
||||
let abort: () => void = () => undefined;
|
||||
const result =
|
||||
effective === 'runs'
|
||||
? this.sendViaRuns(options, (fn) => (abort = fn))
|
||||
: effective === 'sessions'
|
||||
? this.sendViaSessions(options, (fn) => (abort = fn))
|
||||
: this.sendViaCompletions(options, (fn) => (abort = fn));
|
||||
return { result, abort: () => abort(), transport: effective };
|
||||
}
|
||||
|
||||
private async sendViaRuns(options: SendOptions, setAbort: (fn: () => void) => void): Promise<string> {
|
||||
const { onEvent } = options;
|
||||
const run = await this.startRun({
|
||||
input: options.text,
|
||||
session_id: options.sessionId || undefined,
|
||||
instructions: this.config.instructions || undefined,
|
||||
model: this.config.model || undefined
|
||||
});
|
||||
const runId = run.run_id;
|
||||
onEvent({ kind: 'run.started', runId });
|
||||
Log.info('hermes', `run ${runId} started`);
|
||||
|
||||
let stopped = false;
|
||||
let finalText = '';
|
||||
let streamedText = '';
|
||||
let completed = false;
|
||||
let failure: string | null = null;
|
||||
|
||||
const handle = await this.streamRunEvents(runId, (event) => {
|
||||
if (event.kind === 'delta') streamedText += event.text;
|
||||
if (event.kind === 'completed' && event.status !== 'message') {
|
||||
completed = true;
|
||||
finalText = event.text ?? '';
|
||||
}
|
||||
if (event.kind === 'error') failure = event.message;
|
||||
onEvent(event);
|
||||
});
|
||||
setAbort(() => {
|
||||
stopped = true;
|
||||
handle.abort();
|
||||
this.stopRun(runId).catch(() => undefined);
|
||||
});
|
||||
|
||||
await handle.done;
|
||||
if (stopped) return streamedText;
|
||||
if (failure) throw new Error(failure);
|
||||
|
||||
if (!completed) {
|
||||
// The event buffer may have been consumed by a reconnect; fall back to polling the run status.
|
||||
for (let i = 0; i < 120 && !stopped; i++) {
|
||||
const info = await this.getRun(runId);
|
||||
if (['completed', 'failed', 'cancelled', 'canceled', 'stopped'].includes(info.status)) {
|
||||
if (info.status !== 'completed') throw new Error(String(info.error ?? `run ${info.status}`));
|
||||
finalText = info.output ?? '';
|
||||
onEvent({ kind: 'completed', text: finalText, status: info.status, usage: info.usage });
|
||||
if (info.session_id) onEvent({ kind: 'session', sessionId: info.session_id });
|
||||
break;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 1000));
|
||||
}
|
||||
}
|
||||
return finalText || streamedText;
|
||||
}
|
||||
|
||||
private async sendViaSessions(options: SendOptions, setAbort: (fn: () => void) => void): Promise<string> {
|
||||
const { onEvent } = options;
|
||||
let sessionId = options.sessionId;
|
||||
if (!sessionId || !sessionId.startsWith('hs:')) {
|
||||
const session = await this.createSession('EveFlow');
|
||||
sessionId = `hs:${session.id}`;
|
||||
onEvent({ kind: 'session', sessionId });
|
||||
}
|
||||
const realId = sessionId.slice(3);
|
||||
const body: Rec = { input: options.text };
|
||||
if (this.config.instructions) body.instructions = this.config.instructions;
|
||||
if (this.config.model) body.model = this.config.model;
|
||||
|
||||
let streamed = '';
|
||||
let finalText = '';
|
||||
let failure: string | null = null;
|
||||
const handle = await this.openSse(`/api/sessions/${encodeURIComponent(realId)}/chat/stream`, { method: 'POST', body }, (event) => {
|
||||
if (event.kind === 'delta') streamed += event.text;
|
||||
if (event.kind === 'completed' && event.text) finalText = event.text;
|
||||
if (event.kind === 'error') failure = event.message;
|
||||
if (event.kind === 'session') return; // keep our prefixed id
|
||||
onEvent(event);
|
||||
});
|
||||
setAbort(() => handle.abort());
|
||||
await handle.done;
|
||||
if (failure) throw new Error(failure);
|
||||
return finalText || streamed;
|
||||
}
|
||||
|
||||
private async sendViaCompletions(options: SendOptions, setAbort: (fn: () => void) => void): Promise<string> {
|
||||
const { onEvent } = options;
|
||||
const useTools = this.config.localTools && !!options.localToolExecutor && !!options.localToolDefinitions?.length;
|
||||
const messages: HermesChatMessage[] = [];
|
||||
if (this.config.instructions) messages.push({ role: 'system', content: this.config.instructions });
|
||||
for (const h of options.history) messages.push({ role: h.role, content: h.content });
|
||||
const userContent: HermesChatMessage['content'] = options.images?.length
|
||||
? [{ type: 'text', text: options.text }, ...options.images.map((url) => ({ type: 'image_url', image_url: { url } }))]
|
||||
: options.text;
|
||||
messages.push({ role: 'user', content: userContent });
|
||||
|
||||
let aborted = false;
|
||||
let currentHandle: StreamHandle | null = null;
|
||||
setAbort(() => {
|
||||
aborted = true;
|
||||
currentHandle?.abort();
|
||||
});
|
||||
|
||||
let fullText = '';
|
||||
for (let iteration = 0; iteration < 6 && !aborted; iteration++) {
|
||||
const payload: Rec = { model: this.config.model || 'hermes-agent', messages, stream: true };
|
||||
if (useTools) {
|
||||
payload.tools = options.localToolDefinitions;
|
||||
payload.tool_choice = 'auto';
|
||||
}
|
||||
if (this.config.reasoningEffort) payload.model_options = { reasoning_effort: this.config.reasoningEffort };
|
||||
|
||||
const toolCalls = new Map<number, CompletionToolCallAccumulator>();
|
||||
let finishReason: string | null = null;
|
||||
let iterationText = '';
|
||||
const errorChunks: string[] = [];
|
||||
let ready = false;
|
||||
const parser = new SseParser((message) => {
|
||||
if (message.data === '[DONE]') return;
|
||||
const data = tryParseJson<unknown>(message.data);
|
||||
if (data === null) return;
|
||||
if (message.event !== 'message' && !(isRec(data) && Array.isArray(data.choices))) {
|
||||
for (const event of normalizeLifecycleEvent(message.event, data)) onEvent(event);
|
||||
return;
|
||||
}
|
||||
const { events, finishReason: reason } = normalizeCompletionChunk(data, toolCalls);
|
||||
for (const event of events) {
|
||||
if (event.kind === 'delta') iterationText += event.text;
|
||||
onEvent(event);
|
||||
}
|
||||
if (reason) finishReason = reason;
|
||||
});
|
||||
|
||||
const headers = this.headers({ Accept: 'text/event-stream' });
|
||||
if (options.sessionId) headers['X-Hermes-Session-Id'] = options.sessionId;
|
||||
const handle = await httpStream(
|
||||
{ url: `${this.base}/v1/chat/completions`, method: 'POST', headers, body: JSON.stringify(payload), timeoutMs: 10 * 60_000 },
|
||||
{ onChunk: (text) => (ready ? parser.feed(text) : errorChunks.push(text)) }
|
||||
);
|
||||
currentHandle = handle;
|
||||
if (!handle.start.ok) {
|
||||
await handle.done.catch(() => undefined);
|
||||
throw new HttpError(handle.start.status, errorMessage(handle.start.status, errorChunks.join('')));
|
||||
}
|
||||
ready = true;
|
||||
const rotated = handle.start.headers['x-hermes-session-id'];
|
||||
if (rotated && rotated !== options.sessionId) onEvent({ kind: 'session', sessionId: rotated });
|
||||
|
||||
await handle.done;
|
||||
parser.end();
|
||||
fullText += iterationText;
|
||||
|
||||
if (finishReason === 'tool_calls' && useTools && toolCalls.size > 0 && !aborted) {
|
||||
const calls = [...toolCalls.values()];
|
||||
messages.push({
|
||||
role: 'assistant',
|
||||
content: iterationText || null,
|
||||
tool_calls: calls.map((c) => ({ id: c.id, type: 'function', function: { name: c.name, arguments: c.arguments } }))
|
||||
});
|
||||
for (const call of calls) {
|
||||
onEvent({ kind: 'tool.start', id: call.id, name: call.name, args: call.arguments });
|
||||
const output = await options.localToolExecutor!(call.name, call.arguments);
|
||||
onEvent({ kind: 'tool.end', id: call.id, name: call.name, output, ok: !output.includes('"error"') });
|
||||
messages.push({ role: 'tool', content: output, tool_call_id: call.id });
|
||||
}
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
onEvent({ kind: 'completed', status: aborted ? 'cancelled' : 'completed', text: fullText });
|
||||
return fullText;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeJob(job: HermesJob): HermesJob {
|
||||
const raw = job as Rec;
|
||||
const id = String(raw.id ?? raw.job_id ?? raw.name ?? '');
|
||||
return { ...job, id };
|
||||
}
|
||||
|
||||
export function jobStatus(job: HermesJob): string {
|
||||
if (job.state) return String(job.state);
|
||||
if (job.status) return String(job.status);
|
||||
if (job.enabled === false) return 'paused';
|
||||
return 'active';
|
||||
}
|
||||
|
||||
export function jobSchedule(job: HermesJob): string {
|
||||
const s = job.schedule;
|
||||
if (!s) return '—';
|
||||
if (typeof s === 'string') return s;
|
||||
return s.display || s.expr || s.kind || 'schedule';
|
||||
}
|
||||
|
||||
export function jobOutput(job: HermesJob): string {
|
||||
return String(job.last_output ?? job.last_error ?? job.output ?? job.result ?? '');
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
/**
|
||||
* Normalises the various Hermes streaming dialects (runs events, sessions stream,
|
||||
* chat-completions chunks) into one HermesStreamEvent vocabulary.
|
||||
* Hermes releases have renamed a few events (tool.started/tool.start, token.delta/message.delta),
|
||||
* so every reader here is deliberately tolerant.
|
||||
*/
|
||||
import type { HermesStreamEvent, HermesUsage } from './types';
|
||||
|
||||
type Rec = Record<string, unknown>;
|
||||
const isRec = (v: unknown): v is Rec => !!v && typeof v === 'object' && !Array.isArray(v);
|
||||
const s = (v: unknown): string => (typeof v === 'string' ? v : v == null ? '' : typeof v === 'object' ? JSON.stringify(v) : String(v));
|
||||
const first = (obj: Rec, keys: string[]): unknown => {
|
||||
for (const k of keys) if (obj[k] !== undefined && obj[k] !== null) return obj[k];
|
||||
return undefined;
|
||||
};
|
||||
|
||||
function textOf(data: Rec): string {
|
||||
const direct = first(data, ['delta', 'text', 'content', 'output_text', 'token']);
|
||||
if (typeof direct === 'string') return direct;
|
||||
if (isRec(direct)) return s(first(direct, ['text', 'content', 'delta']));
|
||||
return '';
|
||||
}
|
||||
|
||||
function idOf(data: Rec, fallback = ''): string {
|
||||
return s(first(data, ['tool_call_id', 'call_id', 'id', 'request_id', 'delegation_id'])) || fallback;
|
||||
}
|
||||
|
||||
function toolName(data: Rec): string {
|
||||
const direct = first(data, ['tool_name', 'tool', 'name', 'function']);
|
||||
if (typeof direct === 'string') return direct;
|
||||
if (isRec(direct)) return s(first(direct, ['name', 'tool']));
|
||||
return 'outil';
|
||||
}
|
||||
|
||||
function argsOf(data: Rec): string | undefined {
|
||||
const value = first(data, ['arguments', 'args', 'input', 'params', 'preview']);
|
||||
if (value === undefined) return undefined;
|
||||
return typeof value === 'string' ? value : JSON.stringify(value);
|
||||
}
|
||||
|
||||
function optionsOf(data: Rec): string[] | undefined {
|
||||
const value = first(data, ['options', 'choices']);
|
||||
if (!Array.isArray(value)) return undefined;
|
||||
return value.map((v) => (isRec(v) ? s(first(v, ['label', 'value', 'id'])) : s(v))).filter(Boolean);
|
||||
}
|
||||
|
||||
function usageOf(data: Rec): HermesUsage | undefined {
|
||||
return isRec(data.usage) ? (data.usage as HermesUsage) : undefined;
|
||||
}
|
||||
|
||||
/** Runs API + sessions stream share the same lifecycle vocabulary. */
|
||||
export function normalizeLifecycleEvent(event: string, payload: unknown): HermesStreamEvent[] {
|
||||
const name = event.toLowerCase();
|
||||
const data: Rec = isRec(payload) ? payload : { value: payload };
|
||||
const out: HermesStreamEvent[] = [];
|
||||
const sessionId = s(first(data, ['session_id', 'sessionId']));
|
||||
|
||||
switch (name) {
|
||||
case 'token.delta':
|
||||
case 'message.delta':
|
||||
case 'assistant.delta':
|
||||
case 'output_text.delta':
|
||||
case 'response.output_text.delta':
|
||||
case 'delta': {
|
||||
const text = textOf(data);
|
||||
if (text) out.push({ kind: 'delta', text });
|
||||
break;
|
||||
}
|
||||
case 'reasoning.delta':
|
||||
case 'reasoning.available':
|
||||
case 'thinking.delta': {
|
||||
const text = textOf(data);
|
||||
if (text) out.push({ kind: 'reasoning', text });
|
||||
break;
|
||||
}
|
||||
case 'tool.start':
|
||||
case 'tool.started':
|
||||
case 'tool_call.start':
|
||||
out.push({ kind: 'tool.start', id: idOf(data, `tool-${Date.now()}`), name: toolName(data), args: argsOf(data) });
|
||||
break;
|
||||
case 'tool.progress':
|
||||
case 'hermes.tool.progress':
|
||||
out.push({
|
||||
kind: 'tool.progress',
|
||||
id: idOf(data) || undefined,
|
||||
name: toolName(data),
|
||||
label: s(first(data, ['label', 'message', 'status'])) || undefined,
|
||||
detail: s(first(data, ['preview', 'detail', 'output'])) || undefined
|
||||
});
|
||||
break;
|
||||
case 'tool.complete':
|
||||
case 'tool.completed':
|
||||
case 'tool.end':
|
||||
case 'tool_call.complete': {
|
||||
const status = s(first(data, ['status', 'result_status'])).toLowerCase();
|
||||
const error = first(data, ['error']);
|
||||
out.push({
|
||||
kind: 'tool.end',
|
||||
id: idOf(data, ''),
|
||||
name: data.tool_name || data.tool || data.name ? toolName(data) : undefined,
|
||||
output: s(first(data, ['output', 'result', 'content'])) || undefined,
|
||||
ok: !error && status !== 'error' && status !== 'failed'
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'subagent.start':
|
||||
out.push({
|
||||
kind: 'subagent.start',
|
||||
id: s(first(data, ['delegation_id', 'id'])) || `sub-${Date.now()}`,
|
||||
childSessionId: s(data.child_session_id) || undefined,
|
||||
label: s(first(data, ['label', 'task', 'goal'])) || undefined
|
||||
});
|
||||
break;
|
||||
case 'subagent.complete':
|
||||
case 'subagent.end':
|
||||
out.push({
|
||||
kind: 'subagent.end',
|
||||
id: s(first(data, ['delegation_id', 'id'])) || 'sub',
|
||||
status: s(data.status) || 'completed',
|
||||
summary: s(data.summary) || undefined,
|
||||
durationMs: typeof data.duration_ms === 'number' ? data.duration_ms : undefined
|
||||
});
|
||||
break;
|
||||
case 'approval.request':
|
||||
case 'approval.requested':
|
||||
out.push({
|
||||
kind: 'approval.request',
|
||||
requestId: s(first(data, ['request_id', 'approval_id', 'id'])),
|
||||
tool: s(first(data, ['tool_name', 'tool', 'command'])) || undefined,
|
||||
description: s(first(data, ['description', 'message', 'prompt', 'reason', 'command', 'preview'])) || 'Hermes demande une autorisation.',
|
||||
args: argsOf(data),
|
||||
options: optionsOf(data)
|
||||
});
|
||||
break;
|
||||
case 'clarify.request':
|
||||
case 'clarification.request':
|
||||
out.push({
|
||||
kind: 'clarify.request',
|
||||
requestId: s(first(data, ['request_id', 'id'])),
|
||||
question: s(first(data, ['question', 'prompt', 'message'])) || 'Hermes a besoin d’une précision.',
|
||||
options: optionsOf(data)
|
||||
});
|
||||
break;
|
||||
case 'sudo.request':
|
||||
case 'secret.request':
|
||||
out.push({
|
||||
kind: 'input.request',
|
||||
requestId: s(first(data, ['request_id', 'id'])),
|
||||
prompt: s(first(data, ['prompt', 'message', 'description'])) || (name === 'sudo.request' ? 'Mot de passe sudo requis' : 'Secret requis'),
|
||||
secret: true,
|
||||
variant: name === 'sudo.request' ? 'sudo' : 'secret'
|
||||
});
|
||||
break;
|
||||
case 'approval.expire':
|
||||
case 'clarify.expire':
|
||||
case 'sudo.expire':
|
||||
case 'secret.expire':
|
||||
case 'request.expire':
|
||||
out.push({ kind: 'request.expire', requestId: s(first(data, ['request_id', 'id'])) });
|
||||
break;
|
||||
case 'run.started':
|
||||
case 'run.created':
|
||||
out.push({ kind: 'run.started', runId: s(first(data, ['run_id', 'id'])) });
|
||||
break;
|
||||
case 'message.complete':
|
||||
case 'message.completed':
|
||||
case 'assistant.complete': {
|
||||
// Final text; a run.completed usually follows, so only surface the text here.
|
||||
const text = textOf(data);
|
||||
if (text) out.push({ kind: 'completed', text, status: 'message', usage: usageOf(data) });
|
||||
break;
|
||||
}
|
||||
case 'run.completed':
|
||||
case 'run.complete':
|
||||
case 'run.finished':
|
||||
case 'done':
|
||||
case 'complete':
|
||||
out.push({
|
||||
kind: 'completed',
|
||||
text: s(first(data, ['output', 'text', 'content', 'final_output'])) || undefined,
|
||||
status: s(data.status) || 'completed',
|
||||
usage: usageOf(data)
|
||||
});
|
||||
break;
|
||||
case 'run.failed':
|
||||
case 'run.cancelled':
|
||||
case 'run.canceled':
|
||||
case 'error':
|
||||
out.push({ kind: 'error', message: s(first(data, ['error', 'message', 'detail', 'reason'])) || name });
|
||||
break;
|
||||
case 'gateway.ready':
|
||||
case 'ping':
|
||||
case 'heartbeat':
|
||||
break;
|
||||
default:
|
||||
out.push({ kind: 'raw', event, data: payload });
|
||||
}
|
||||
if (sessionId) out.push({ kind: 'session', sessionId });
|
||||
return out;
|
||||
}
|
||||
|
||||
export interface CompletionToolCallAccumulator {
|
||||
id: string;
|
||||
name: string;
|
||||
arguments: string;
|
||||
}
|
||||
|
||||
/** Chat-completions chunk (OpenAI format). Returns delta events plus any accumulated tool calls. */
|
||||
export function normalizeCompletionChunk(
|
||||
chunk: unknown,
|
||||
toolCalls: Map<number, CompletionToolCallAccumulator>
|
||||
): { events: HermesStreamEvent[]; finishReason: string | null } {
|
||||
const events: HermesStreamEvent[] = [];
|
||||
if (!isRec(chunk)) return { events, finishReason: null };
|
||||
const choice = Array.isArray(chunk.choices) && isRec(chunk.choices[0]) ? (chunk.choices[0] as Rec) : null;
|
||||
if (!choice) return { events, finishReason: null };
|
||||
const delta = isRec(choice.delta) ? (choice.delta as Rec) : isRec(choice.message) ? (choice.message as Rec) : null;
|
||||
if (delta) {
|
||||
if (typeof delta.content === 'string' && delta.content) events.push({ kind: 'delta', text: delta.content });
|
||||
const reasoning = first(delta, ['reasoning_content', 'reasoning']);
|
||||
if (typeof reasoning === 'string' && reasoning) events.push({ kind: 'reasoning', text: reasoning });
|
||||
if (Array.isArray(delta.tool_calls)) {
|
||||
for (const tc of delta.tool_calls) {
|
||||
if (!isRec(tc)) continue;
|
||||
const index = typeof tc.index === 'number' ? tc.index : toolCalls.size;
|
||||
const fn = isRec(tc.function) ? (tc.function as Rec) : {};
|
||||
const acc = toolCalls.get(index) ?? { id: s(tc.id), name: '', arguments: '' };
|
||||
if (tc.id) acc.id = s(tc.id);
|
||||
if (typeof fn.name === 'string') acc.name += fn.name;
|
||||
if (typeof fn.arguments === 'string') acc.arguments += fn.arguments;
|
||||
toolCalls.set(index, acc);
|
||||
}
|
||||
}
|
||||
}
|
||||
return { events, finishReason: typeof choice.finish_reason === 'string' ? choice.finish_reason : null };
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
/**
|
||||
* Client-side tools offered to Hermes in chat-completions mode (OpenAI tool calling).
|
||||
* With the runs/sessions transports Hermes uses its own server-side toolsets instead.
|
||||
*/
|
||||
import { bridge } from '../../lib/bridge';
|
||||
|
||||
export interface LocalToolContext {
|
||||
setEmotion: (emotion: string) => void;
|
||||
speak: (text: string) => void;
|
||||
getStatus: () => Record<string, unknown>;
|
||||
getHistory: (n: number) => Array<{ role: string; content: string }>;
|
||||
notify: (title: string, body: string) => void;
|
||||
}
|
||||
|
||||
export const LOCAL_TOOL_DEFINITIONS = [
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'set_hud_state',
|
||||
description: "Change l'état visuel du HUD EveFlow (couleur/animation du noyau) pour accompagner la réponse.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
state: { type: 'string', enum: ['neutral', 'happy', 'thinking', 'alert', 'error', 'success'] }
|
||||
},
|
||||
required: ['state']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_app_status',
|
||||
description: "Retourne l'état courant d'EveFlow (transport, voix, nombre de messages, hôte).",
|
||||
parameters: { type: 'object', properties: {} }
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_conversation_history',
|
||||
description: 'Retourne les N derniers messages affichés dans EveFlow.',
|
||||
parameters: { type: 'object', properties: { n: { type: 'number', minimum: 1, maximum: 30 } }, required: ['n'] }
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'speak_text',
|
||||
description: 'Fait prononcer immédiatement un texte court par la voix du HUD.',
|
||||
parameters: { type: 'object', properties: { text: { type: 'string' } }, required: ['text'] }
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'notify_user',
|
||||
description: 'Affiche une notification système Windows à l’utilisateur.',
|
||||
parameters: { type: 'object', properties: { title: { type: 'string' }, body: { type: 'string' } }, required: ['title', 'body'] }
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'write_shared_file',
|
||||
description: "Écrit un fichier texte (rapport, script, JSON, HTML, SVG) dans le dossier partagé EveFlow_Shared de l'utilisateur et renvoie son URL.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: { filename: { type: 'string' }, content: { type: 'string' } },
|
||||
required: ['filename', 'content']
|
||||
}
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
export async function executeLocalTool(name: string, rawArgs: string, ctx: LocalToolContext): Promise<string> {
|
||||
let args: Record<string, unknown> = {};
|
||||
try {
|
||||
args = rawArgs ? (JSON.parse(rawArgs) as Record<string, unknown>) : {};
|
||||
} catch {
|
||||
return JSON.stringify({ error: 'invalid JSON arguments' });
|
||||
}
|
||||
try {
|
||||
switch (name) {
|
||||
case 'set_hud_state':
|
||||
ctx.setEmotion(String(args.state ?? 'neutral'));
|
||||
return JSON.stringify({ ok: true });
|
||||
case 'get_app_status':
|
||||
return JSON.stringify(ctx.getStatus());
|
||||
case 'get_conversation_history':
|
||||
return JSON.stringify({ messages: ctx.getHistory(Math.min(30, Number(args.n) || 5)) });
|
||||
case 'speak_text':
|
||||
ctx.speak(String(args.text ?? '').slice(0, 600));
|
||||
return JSON.stringify({ ok: true });
|
||||
case 'notify_user':
|
||||
ctx.notify(String(args.title ?? 'EveFlow'), String(args.body ?? ''));
|
||||
return JSON.stringify({ ok: true });
|
||||
case 'write_shared_file': {
|
||||
const api = bridge();
|
||||
if (!api) return JSON.stringify({ error: 'file system unavailable outside Electron' });
|
||||
const result = await api.files.writeShared(String(args.filename ?? 'fichier.txt'), String(args.content ?? ''), false);
|
||||
return JSON.stringify({ ok: true, ...result });
|
||||
}
|
||||
default:
|
||||
return JSON.stringify({ error: `unknown tool ${name}` });
|
||||
}
|
||||
} catch (err) {
|
||||
return JSON.stringify({ error: (err as Error).message });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
export type HermesTransport = 'auto' | 'runs' | 'sessions' | 'completions';
|
||||
|
||||
export interface HermesConfig {
|
||||
url: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
/** Stable per-user key for long-term memory (X-Hermes-Session-Key). */
|
||||
sessionKey: string;
|
||||
transport: HermesTransport;
|
||||
reasoningEffort: '' | 'low' | 'medium' | 'high';
|
||||
/** Extra instructions layered on top of the Hermes system prompt. */
|
||||
instructions: string;
|
||||
/** Expose EveFlow client tools in chat-completions mode. */
|
||||
localTools: boolean;
|
||||
}
|
||||
|
||||
export interface HermesCapabilities {
|
||||
object?: string;
|
||||
platform?: string;
|
||||
model?: string;
|
||||
version?: string;
|
||||
auth?: { type?: string; required?: boolean };
|
||||
features?: Record<string, boolean | string>;
|
||||
endpoints?: Record<string, boolean>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesHealth {
|
||||
status: string;
|
||||
readiness?: { checks?: Record<string, unknown> };
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesModel {
|
||||
id: string;
|
||||
owned_by?: string;
|
||||
provider?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesSkill {
|
||||
name: string;
|
||||
description?: string;
|
||||
category?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesToolset {
|
||||
name: string;
|
||||
label?: string;
|
||||
description?: string;
|
||||
enabled?: boolean;
|
||||
configured?: boolean;
|
||||
tools?: string[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesSession {
|
||||
id: string;
|
||||
title?: string;
|
||||
created_at?: string | number;
|
||||
updated_at?: string | number;
|
||||
end_reason?: string | null;
|
||||
message_count?: number;
|
||||
platform?: string;
|
||||
source?: string;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesSessionMessage {
|
||||
role: string;
|
||||
content: string;
|
||||
timestamp?: string | number;
|
||||
tool_calls?: unknown[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesJobSchedule {
|
||||
kind?: string;
|
||||
expr?: string;
|
||||
display?: string;
|
||||
}
|
||||
|
||||
export interface HermesJob {
|
||||
id: string;
|
||||
name?: string;
|
||||
prompt?: string;
|
||||
schedule?: string | HermesJobSchedule;
|
||||
state?: string;
|
||||
status?: string;
|
||||
enabled?: boolean;
|
||||
deliver?: string;
|
||||
next_run_at?: string | number;
|
||||
last_run_at?: string | number;
|
||||
last_status?: string;
|
||||
last_output?: string;
|
||||
last_error?: string;
|
||||
repeat?: number | { times?: number | null; completed?: number };
|
||||
skills?: string[];
|
||||
skill?: string;
|
||||
provider?: string | null;
|
||||
model?: string | null;
|
||||
created_at?: string | number;
|
||||
updated_at?: string | number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesJobDraft {
|
||||
name: string;
|
||||
schedule: string;
|
||||
prompt: string;
|
||||
deliver?: string;
|
||||
skills?: string[];
|
||||
}
|
||||
|
||||
export interface HermesRunInfo {
|
||||
run_id: string;
|
||||
status: string;
|
||||
session_id?: string;
|
||||
model?: string;
|
||||
output?: string;
|
||||
usage?: Record<string, number>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesUsage {
|
||||
input_tokens?: number;
|
||||
output_tokens?: number;
|
||||
total_tokens?: number;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export type HermesStreamEvent =
|
||||
| { kind: 'delta'; text: string }
|
||||
| { kind: 'reasoning'; text: string }
|
||||
| { kind: 'tool.start'; id: string; name: string; args?: string }
|
||||
| { kind: 'tool.progress'; id?: string; name: string; label?: string; detail?: string }
|
||||
| { kind: 'tool.end'; id: string; name?: string; output?: string; ok: boolean }
|
||||
| { kind: 'subagent.start'; id: string; childSessionId?: string; label?: string }
|
||||
| { kind: 'subagent.end'; id: string; status: string; summary?: string; durationMs?: number }
|
||||
| { kind: 'approval.request'; requestId: string; tool?: string; description: string; args?: string; options?: string[] }
|
||||
| { kind: 'clarify.request'; requestId: string; question: string; options?: string[] }
|
||||
| { kind: 'input.request'; requestId: string; prompt: string; secret: boolean; variant: 'sudo' | 'secret' }
|
||||
| { kind: 'request.expire'; requestId: string }
|
||||
| { kind: 'session'; sessionId: string }
|
||||
| { kind: 'run.started'; runId: string }
|
||||
| { kind: 'completed'; text?: string; status: string; usage?: HermesUsage }
|
||||
| { kind: 'error'; message: string }
|
||||
| { kind: 'raw'; event: string; data: unknown };
|
||||
|
||||
export interface HermesChatMessage {
|
||||
role: 'user' | 'assistant' | 'system' | 'tool';
|
||||
content: string | null | Array<{ type: string; text?: string; image_url?: { url: string } }>;
|
||||
tool_call_id?: string;
|
||||
tool_calls?: unknown[];
|
||||
}
|
||||
|
||||
export interface SendOptions {
|
||||
text: string;
|
||||
images?: string[];
|
||||
/** Recent turns used only by the stateless chat-completions transport. */
|
||||
history: Array<{ role: 'user' | 'assistant'; content: string }>;
|
||||
sessionId: string;
|
||||
onEvent: (event: HermesStreamEvent) => void;
|
||||
localToolExecutor?: (name: string, args: string) => Promise<string>;
|
||||
localToolDefinitions?: unknown[];
|
||||
}
|
||||
|
||||
export interface SendHandle {
|
||||
/** Resolves with the final assistant text. */
|
||||
result: Promise<string>;
|
||||
abort: () => void;
|
||||
transport: Exclude<HermesTransport, 'auto'>;
|
||||
}
|
||||
@@ -1,221 +0,0 @@
|
||||
/**
|
||||
* PollingService — synchronizes Hermes /api/jobs for scheduled work.
|
||||
*
|
||||
* Hermes remains the source of truth. EveFlow polls as a client, keeps a local
|
||||
* display cache, and uses the same service for CRUD actions on remote jobs.
|
||||
*/
|
||||
|
||||
import { LogService } from './logService';
|
||||
|
||||
export type HermesJobStatus =
|
||||
| 'pending'
|
||||
| 'scheduled'
|
||||
| 'paused'
|
||||
| 'running'
|
||||
| 'completed'
|
||||
| 'failed'
|
||||
| 'ok'
|
||||
| string;
|
||||
|
||||
export interface HermesJob {
|
||||
id: string;
|
||||
name?: string;
|
||||
prompt?: string;
|
||||
schedule?: string | { kind?: string; expr?: string; display?: string };
|
||||
status?: HermesJobStatus;
|
||||
state?: HermesJobStatus;
|
||||
enabled?: boolean;
|
||||
result?: string;
|
||||
output?: string;
|
||||
last_result?: string;
|
||||
last_output?: string;
|
||||
error?: string;
|
||||
last_error?: string;
|
||||
created_at?: string;
|
||||
updated_at?: string;
|
||||
next_run_at?: string;
|
||||
last_run_at?: string;
|
||||
completed_at?: string;
|
||||
last_status?: string;
|
||||
repeat?: { times?: number | null; completed?: number };
|
||||
skills?: string[];
|
||||
deliver?: string;
|
||||
provider?: string | null;
|
||||
model?: string | null;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface HermesJobDraft {
|
||||
name: string;
|
||||
schedule: string;
|
||||
prompt: string;
|
||||
deliver?: string;
|
||||
}
|
||||
|
||||
export interface HermesSyncSnapshot {
|
||||
jobs: HermesJob[];
|
||||
syncedAt: string;
|
||||
}
|
||||
|
||||
export type JobsSyncHandler = (snapshot: HermesSyncSnapshot) => void;
|
||||
export type SyncStatusHandler = (status: 'syncing' | 'online' | 'offline', detail?: string) => void;
|
||||
|
||||
const normalizeBaseUrl = (hermesUrl: string): string => (
|
||||
hermesUrl.replace(/\/$/, '').replace(/\/v1(\/.*)?$/, '')
|
||||
);
|
||||
|
||||
const parseJobsPayload = (payload: unknown): HermesJob[] => {
|
||||
if (Array.isArray(payload)) return payload as HermesJob[];
|
||||
if (payload && typeof payload === 'object') {
|
||||
const maybeJobs = (payload as { jobs?: unknown }).jobs;
|
||||
if (Array.isArray(maybeJobs)) return maybeJobs as HermesJob[];
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
export class PollingService {
|
||||
private intervalId: ReturnType<typeof setInterval> | null = null;
|
||||
private hermesUrl: string = '';
|
||||
private hermesApiKey: string = '';
|
||||
private onSync: JobsSyncHandler | null = null;
|
||||
private onStatus: SyncStatusHandler | null = null;
|
||||
private intervalMs: number;
|
||||
|
||||
constructor(intervalMs: number = 30_000) {
|
||||
this.intervalMs = intervalMs;
|
||||
}
|
||||
|
||||
public start(
|
||||
hermesUrl: string,
|
||||
apiKey: string,
|
||||
onSync: JobsSyncHandler,
|
||||
onStatus?: SyncStatusHandler
|
||||
): void {
|
||||
this.stop();
|
||||
this.hermesUrl = hermesUrl;
|
||||
this.hermesApiKey = apiKey;
|
||||
this.onSync = onSync;
|
||||
this.onStatus = onStatus || null;
|
||||
|
||||
LogService.info('PollingService', `Synchronisation Hermes jobs toutes les ${this.intervalMs / 1000}s`, { hermesUrl });
|
||||
|
||||
this.sync();
|
||||
this.intervalId = setInterval(() => this.sync(), this.intervalMs);
|
||||
}
|
||||
|
||||
public stop(): void {
|
||||
if (this.intervalId !== null) {
|
||||
clearInterval(this.intervalId);
|
||||
this.intervalId = null;
|
||||
LogService.info('PollingService', 'Synchronisation Hermes jobs arrêtée.');
|
||||
}
|
||||
}
|
||||
|
||||
public get isActive(): boolean {
|
||||
return this.intervalId !== null;
|
||||
}
|
||||
|
||||
public async refresh(): Promise<HermesSyncSnapshot> {
|
||||
return this.fetchJobs();
|
||||
}
|
||||
|
||||
public async createJob(draft: HermesJobDraft): Promise<HermesJob> {
|
||||
return this.requestJob('', {
|
||||
method: 'POST',
|
||||
body: JSON.stringify({
|
||||
name: draft.name,
|
||||
schedule: draft.schedule,
|
||||
prompt: draft.prompt,
|
||||
...(draft.deliver ? { deliver: draft.deliver } : {})
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
public async updateJob(jobId: string, patch: Partial<HermesJobDraft>): Promise<HermesJob> {
|
||||
return this.requestJob(`/${encodeURIComponent(jobId)}`, {
|
||||
method: 'PATCH',
|
||||
body: JSON.stringify(patch)
|
||||
});
|
||||
}
|
||||
|
||||
public async deleteJob(jobId: string): Promise<void> {
|
||||
await this.requestRaw(`/${encodeURIComponent(jobId)}`, { method: 'DELETE' });
|
||||
}
|
||||
|
||||
public async pauseJob(jobId: string): Promise<HermesJob> {
|
||||
return this.requestJob(`/${encodeURIComponent(jobId)}/pause`, { method: 'POST' });
|
||||
}
|
||||
|
||||
public async resumeJob(jobId: string): Promise<HermesJob> {
|
||||
return this.requestJob(`/${encodeURIComponent(jobId)}/resume`, { method: 'POST' });
|
||||
}
|
||||
|
||||
public async runJob(jobId: string): Promise<HermesJob> {
|
||||
return this.requestJob(`/${encodeURIComponent(jobId)}/run`, { method: 'POST' });
|
||||
}
|
||||
|
||||
private async sync(): Promise<void> {
|
||||
if (!this.hermesUrl || !this.onSync) return;
|
||||
|
||||
try {
|
||||
this.onStatus?.('syncing');
|
||||
const snapshot = await this.fetchJobs();
|
||||
this.onSync(snapshot);
|
||||
this.onStatus?.('online');
|
||||
} catch (e: any) {
|
||||
const message = e?.message || String(e);
|
||||
LogService.warn('PollingService', `Synchronisation Hermes jobs impossible: ${message}`);
|
||||
this.onStatus?.('offline', message);
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchJobs(): Promise<HermesSyncSnapshot> {
|
||||
const payload = await this.requestRaw('');
|
||||
return {
|
||||
jobs: parseJobsPayload(payload),
|
||||
syncedAt: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
private async requestJob(path: string, init: RequestInit): Promise<HermesJob> {
|
||||
const payload = await this.requestRaw(path, init);
|
||||
if (payload && typeof payload === 'object' && 'job' in payload) {
|
||||
return (payload as { job: HermesJob }).job;
|
||||
}
|
||||
return payload as HermesJob;
|
||||
}
|
||||
|
||||
private async requestRaw(path: string, init: RequestInit = {}): Promise<unknown> {
|
||||
if (!this.hermesUrl) throw new Error('URL Hermes manquante');
|
||||
|
||||
const base = normalizeBaseUrl(this.hermesUrl);
|
||||
const endpoint = `${base}/api/jobs${path}`;
|
||||
const abortCtrl = new AbortController();
|
||||
const timeoutId = setTimeout(() => abortCtrl.abort(), 12_000);
|
||||
|
||||
try {
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (this.hermesApiKey) headers.Authorization = `Bearer ${this.hermesApiKey.trim()}`;
|
||||
|
||||
const response = await fetch(endpoint, {
|
||||
...init,
|
||||
headers: { ...headers, ...(init.headers || {}) },
|
||||
signal: abortCtrl.signal
|
||||
});
|
||||
clearTimeout(timeoutId);
|
||||
|
||||
if (!response.ok) {
|
||||
const raw = await response.text().catch(() => '');
|
||||
throw new Error(`HTTP ${response.status}${raw ? `: ${raw.slice(0, 180)}` : ''}`);
|
||||
}
|
||||
|
||||
if (response.status === 204) return null;
|
||||
const text = await response.text();
|
||||
return text ? JSON.parse(text) : null;
|
||||
} catch (e: any) {
|
||||
clearTimeout(timeoutId);
|
||||
if (e?.name === 'AbortError') throw new Error('Timeout Hermes /api/jobs');
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,247 +0,0 @@
|
||||
/**
|
||||
* ToolRegistry — Defines all functions exposed to Hermes Agent via OpenAI tool calling.
|
||||
*
|
||||
* Each tool has:
|
||||
* - A JSON Schema definition sent to Hermes in the `tools[]` payload
|
||||
* - A local executor that runs when Hermes invokes it
|
||||
*
|
||||
* AppCallbacks provides access to React state setters from AgentService (stateless).
|
||||
*/
|
||||
|
||||
export interface AppCallbacks {
|
||||
/** Change the avatar's current emotion */
|
||||
setEmotion: (emotion: string) => void;
|
||||
/** Get the current application status snapshot */
|
||||
getStatus: () => AppStatus;
|
||||
/** Get the N most recent chat messages */
|
||||
getHistory: (n: number) => { role: string; content: string }[];
|
||||
/** Trigger TTS speech */
|
||||
speak: (text: string) => void;
|
||||
}
|
||||
|
||||
export interface AppStatus {
|
||||
provider: string;
|
||||
avatar: string;
|
||||
emotion: string;
|
||||
isSpeaking: boolean;
|
||||
isMuted: boolean;
|
||||
messageCount: number;
|
||||
}
|
||||
|
||||
/** OpenAI-compatible tool definition */
|
||||
export interface ToolDefinition {
|
||||
type: 'function';
|
||||
function: {
|
||||
name: string;
|
||||
description: string;
|
||||
parameters: Record<string, unknown>;
|
||||
};
|
||||
}
|
||||
|
||||
/** Result returned to Hermes after tool execution */
|
||||
export interface ToolResult {
|
||||
tool_call_id: string;
|
||||
role: 'tool';
|
||||
content: string;
|
||||
}
|
||||
|
||||
// ── Tool Definitions (sent to Hermes in request payload) ───────────────────
|
||||
|
||||
export const HERMES_TOOLS: ToolDefinition[] = [
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'set_avatar_emotion',
|
||||
description: "Change l'état émotionnel de l'avatar Eve visible par l'utilisateur. Appelle cet outil quand la réponse doit être accompagnée d'une expression visible.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
emotion: {
|
||||
type: 'string',
|
||||
enum: ['neutral', 'happy', 'thinking', 'excited', 'sad', 'angry', 'surprised'],
|
||||
description: "L'émotion à afficher sur l'avatar"
|
||||
}
|
||||
},
|
||||
required: ['emotion']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_app_status',
|
||||
description: "Retourne l'état actuel de l'application EveFlow : provider IA actif, avatar sélectionné, émotion courante, mode audio.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {},
|
||||
required: []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'get_conversation_history',
|
||||
description: 'Retourne les N derniers messages de la conversation en cours.',
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
n: {
|
||||
type: 'number',
|
||||
description: 'Nombre de messages à récupérer (max 20)',
|
||||
minimum: 1,
|
||||
maximum: 20
|
||||
}
|
||||
},
|
||||
required: ['n']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'speak_text',
|
||||
description: "Fait prononcer un texte par la synthèse vocale de l'avatar. Utile pour mettre en avant une information importante.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
text: {
|
||||
type: 'string',
|
||||
description: 'Le texte à prononcer (max 500 caractères)'
|
||||
}
|
||||
},
|
||||
required: ['text']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'write_shared_file',
|
||||
description: "Sauvegarde un fichier (rapport, script python, javascript, données JSON, HTML) dans le dossier partagé EveFlow de l'utilisateur Windows et lui partage un lien d'accès. Utilise cet outil chaque fois que l'utilisateur te demande de générer, d'écrire ou de sauvegarder un fichier sur son ordinateur.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
filename: {
|
||||
type: 'string',
|
||||
description: "Le nom du fichier avec son extension (ex: rapport.txt, script.py, data.json, index.html)"
|
||||
},
|
||||
content: {
|
||||
type: 'string',
|
||||
description: "Le contenu textuel intégral à écrire dans le fichier"
|
||||
}
|
||||
},
|
||||
required: ['filename', 'content']
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'function',
|
||||
function: {
|
||||
name: 'generate_shared_image',
|
||||
description: "Génère un schéma technique, un dessin vectoriel, ou une illustration de haute qualité au format SVG, l'enregistre sur le disque et l'affiche dans le chat. Utilise cet outil dès que l'utilisateur te demande d'afficher, de dessiner ou de générer un graphique, un diagramme ou une image.",
|
||||
parameters: {
|
||||
type: 'object',
|
||||
properties: {
|
||||
filename: {
|
||||
type: 'string',
|
||||
description: "Le nom du fichier SVG (ex: architecture.svg, graphique.svg)"
|
||||
},
|
||||
svgContent: {
|
||||
type: 'string',
|
||||
description: "Le code source SVG complet et valide (incluant les balises <svg>, viewBox, styles et éléments géométriques)"
|
||||
}
|
||||
},
|
||||
required: ['filename', 'svgContent']
|
||||
}
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
// ── Tool Dispatcher ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Executes a tool call requested by Hermes Agent and returns the stringified result.
|
||||
*
|
||||
* @param toolCallId - The ID from Hermes tool_call (must be echoed back in role:tool message)
|
||||
* @param name - Function name requested by Hermes
|
||||
* @param rawArgs - JSON string of arguments from Hermes
|
||||
* @param callbacks - App state accessors injected from the React layer
|
||||
* @returns - ToolResult to append to the messages array
|
||||
*/
|
||||
export async function executeTool(
|
||||
toolCallId: string,
|
||||
name: string,
|
||||
rawArgs: string,
|
||||
callbacks: AppCallbacks
|
||||
): Promise<ToolResult> {
|
||||
let result: string;
|
||||
|
||||
try {
|
||||
const args = JSON.parse(rawArgs || '{}');
|
||||
|
||||
switch (name) {
|
||||
case 'set_avatar_emotion': {
|
||||
const emotion = args.emotion as string;
|
||||
callbacks.setEmotion(emotion);
|
||||
result = JSON.stringify({ success: true, emotion_set: emotion });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'get_app_status': {
|
||||
const status = callbacks.getStatus();
|
||||
result = JSON.stringify(status);
|
||||
break;
|
||||
}
|
||||
|
||||
case 'get_conversation_history': {
|
||||
const n = Math.min(Number(args.n) || 5, 20);
|
||||
const history = callbacks.getHistory(n);
|
||||
result = JSON.stringify({ messages: history, count: history.length });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'speak_text': {
|
||||
const text = String(args.text || '').slice(0, 500);
|
||||
callbacks.speak(text);
|
||||
result = JSON.stringify({ success: true, text_spoken: text });
|
||||
break;
|
||||
}
|
||||
|
||||
case 'write_shared_file': {
|
||||
const filename = String(args.filename || '');
|
||||
const content = String(args.content || '');
|
||||
if ((window as any).electronAPI?.writeSharedFile) {
|
||||
const fileUrl = await (window as any).electronAPI.writeSharedFile(filename, content, false);
|
||||
result = JSON.stringify({ success: true, url: fileUrl, filename });
|
||||
} else {
|
||||
result = JSON.stringify({ error: "L'API d'écriture de fichier local n'est pas disponible (hors Electron)." });
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case 'generate_shared_image': {
|
||||
const filename = String(args.filename || '');
|
||||
const svgContent = String(args.svgContent || '');
|
||||
if ((window as any).electronAPI?.writeSharedFile) {
|
||||
const fileUrl = await (window as any).electronAPI.writeSharedFile(filename, svgContent, false);
|
||||
result = JSON.stringify({ success: true, url: fileUrl, filename });
|
||||
} else {
|
||||
result = JSON.stringify({ error: "L'API de génération d'image locale n'est pas disponible." });
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
default:
|
||||
result = JSON.stringify({ error: `Unknown tool: ${name}` });
|
||||
}
|
||||
} catch (e: any) {
|
||||
result = JSON.stringify({ error: `Tool execution failed: ${e.message}` });
|
||||
}
|
||||
|
||||
return {
|
||||
tool_call_id: toolCallId,
|
||||
role: 'tool',
|
||||
content: result
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
/**
|
||||
* Shared Web Audio graph. Everything that is played (TTS) or captured (mic) passes through
|
||||
* analysers so the JARVIS core can react to the real signal.
|
||||
*/
|
||||
class AudioBus {
|
||||
private ctx: AudioContext | null = null;
|
||||
private outputGain: GainNode | null = null;
|
||||
private outputAnalyser: AnalyserNode | null = null;
|
||||
private inputAnalyser: AnalyserNode | null = null;
|
||||
private timeData = new Float32Array(1024);
|
||||
private freqData = new Uint8Array(512);
|
||||
private synthetic = 0;
|
||||
|
||||
get context(): AudioContext {
|
||||
if (!this.ctx) {
|
||||
this.ctx = new AudioContext({ latencyHint: 'interactive' });
|
||||
this.outputGain = this.ctx.createGain();
|
||||
this.outputAnalyser = this.ctx.createAnalyser();
|
||||
this.outputAnalyser.fftSize = 1024;
|
||||
this.outputAnalyser.smoothingTimeConstant = 0.6;
|
||||
this.outputGain.connect(this.outputAnalyser);
|
||||
this.outputAnalyser.connect(this.ctx.destination);
|
||||
}
|
||||
return this.ctx;
|
||||
}
|
||||
|
||||
async resume(): Promise<void> {
|
||||
const ctx = this.context;
|
||||
if (ctx.state === 'suspended') await ctx.resume().catch(() => undefined);
|
||||
}
|
||||
|
||||
get output(): AudioNode {
|
||||
this.context;
|
||||
return this.outputGain!;
|
||||
}
|
||||
|
||||
setVolume(volume: number): void {
|
||||
this.context;
|
||||
this.outputGain!.gain.value = Math.max(0, Math.min(1.5, volume));
|
||||
}
|
||||
|
||||
/** Attach an external analyser (mic capture) so the HUD can read the input level. */
|
||||
setInputAnalyser(analyser: AnalyserNode | null): void {
|
||||
this.inputAnalyser = analyser;
|
||||
}
|
||||
|
||||
/** For engines that bypass Web Audio (system speech synthesis), fake a plausible level. */
|
||||
setSyntheticLevel(level: number): void {
|
||||
this.synthetic = level;
|
||||
}
|
||||
|
||||
private rms(analyser: AnalyserNode): number {
|
||||
if (this.timeData.length !== analyser.fftSize) this.timeData = new Float32Array(analyser.fftSize);
|
||||
analyser.getFloatTimeDomainData(this.timeData);
|
||||
let sum = 0;
|
||||
for (let i = 0; i < this.timeData.length; i++) sum += this.timeData[i] * this.timeData[i];
|
||||
return Math.sqrt(sum / this.timeData.length);
|
||||
}
|
||||
|
||||
/** Output loudness 0..1 (TTS). */
|
||||
outputLevel(): number {
|
||||
if (this.synthetic > 0) return this.synthetic;
|
||||
if (!this.outputAnalyser) return 0;
|
||||
return Math.min(1, this.rms(this.outputAnalyser) * 4);
|
||||
}
|
||||
|
||||
/** Input loudness 0..1 (microphone). */
|
||||
inputLevel(): number {
|
||||
if (!this.inputAnalyser) return 0;
|
||||
return Math.min(1, this.rms(this.inputAnalyser) * 6);
|
||||
}
|
||||
|
||||
/** Normalised output spectrum (0..1) with `bins` values, low to high frequency. */
|
||||
spectrum(bins: number, source: 'output' | 'input' = 'output'): Float32Array {
|
||||
const analyser = source === 'output' ? this.outputAnalyser : this.inputAnalyser;
|
||||
const out = new Float32Array(bins);
|
||||
if (!analyser) {
|
||||
if (this.synthetic > 0) {
|
||||
const t = performance.now() / 1000;
|
||||
for (let i = 0; i < bins; i++) {
|
||||
out[i] = Math.max(0, this.synthetic * (0.6 + 0.4 * Math.sin(t * 9 + i * 0.7)) * (1 - i / bins));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
if (this.freqData.length !== analyser.frequencyBinCount) this.freqData = new Uint8Array(analyser.frequencyBinCount);
|
||||
analyser.getByteFrequencyData(this.freqData);
|
||||
// Only the lower half of the spectrum carries voice energy.
|
||||
const usable = Math.floor(this.freqData.length * 0.5);
|
||||
for (let i = 0; i < bins; i++) {
|
||||
const from = Math.floor((i / bins) * usable);
|
||||
const to = Math.max(from + 1, Math.floor(((i + 1) / bins) * usable));
|
||||
let sum = 0;
|
||||
for (let j = from; j < to; j++) sum += this.freqData[j];
|
||||
out[i] = sum / (to - from) / 255;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
||||
export const audioBus = new AudioBus();
|
||||
@@ -0,0 +1,261 @@
|
||||
/**
|
||||
* Microphone capture built on AudioWorklet (no deprecated ScriptProcessor, no mic monitoring
|
||||
* feedback). Produces 16 kHz WAV utterances, with energy VAD for hands-free auto stop.
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { audioBus } from './audioBus';
|
||||
import { DEFAULT_VAD, EnergyVad, type VadOptions, type VadSignal } from './vad';
|
||||
import { buildWav16k, rms, type WavResult } from './wav';
|
||||
|
||||
const WORKLET_SOURCE = `
|
||||
class EveFlowCaptureProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.buffer = new Float32Array(2048);
|
||||
this.offset = 0;
|
||||
}
|
||||
process(inputs) {
|
||||
const input = inputs[0];
|
||||
if (!input || input.length === 0) return true;
|
||||
const channel = input[0];
|
||||
if (!channel) return true;
|
||||
let i = 0;
|
||||
while (i < channel.length) {
|
||||
const n = Math.min(channel.length - i, this.buffer.length - this.offset);
|
||||
this.buffer.set(channel.subarray(i, i + n), this.offset);
|
||||
this.offset += n;
|
||||
i += n;
|
||||
if (this.offset === this.buffer.length) {
|
||||
this.port.postMessage(this.buffer, [this.buffer.buffer]);
|
||||
this.buffer = new Float32Array(2048);
|
||||
this.offset = 0;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('eveflow-capture', EveFlowCaptureProcessor);
|
||||
`;
|
||||
|
||||
export type CaptureMode = 'auto' | 'manual';
|
||||
|
||||
export interface CaptureCallbacks {
|
||||
onLevel?: (level: number) => void;
|
||||
onSpeechStart?: () => void;
|
||||
onUtterance: (wav: WavResult) => void;
|
||||
onEnd: (reason: 'stopped' | 'speech-end' | 'no-speech' | 'too-short' | 'max-length' | 'error') => void;
|
||||
onError?: (message: string) => void;
|
||||
}
|
||||
|
||||
export interface CaptureOptions {
|
||||
deviceId?: string;
|
||||
mode: CaptureMode;
|
||||
vad?: Partial<VadOptions>;
|
||||
callbacks: CaptureCallbacks;
|
||||
}
|
||||
|
||||
export interface MicDevice {
|
||||
deviceId: string;
|
||||
label: string;
|
||||
}
|
||||
|
||||
let workletUrl: string | null = null;
|
||||
|
||||
function getWorkletUrl(): string {
|
||||
if (!workletUrl) workletUrl = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'application/javascript' }));
|
||||
return workletUrl;
|
||||
}
|
||||
|
||||
export async function listMicrophones(): Promise<MicDevice[]> {
|
||||
try {
|
||||
const devices = await navigator.mediaDevices.enumerateDevices();
|
||||
return devices
|
||||
.filter((d) => d.kind === 'audioinput')
|
||||
.map((d, i) => ({ deviceId: d.deviceId, label: d.label || `Microphone ${i + 1}` }));
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export class MicCapture {
|
||||
private ctx: AudioContext | null = null;
|
||||
private stream: MediaStream | null = null;
|
||||
private worklet: AudioWorkletNode | null = null;
|
||||
private legacy: ScriptProcessorNode | null = null;
|
||||
private chunks: Float32Array[] = [];
|
||||
private totalSamples = 0;
|
||||
private vad: EnergyVad | null = null;
|
||||
private active = false;
|
||||
private options: CaptureOptions | null = null;
|
||||
private sampleRate = 16_000;
|
||||
|
||||
get isActive(): boolean {
|
||||
return this.active;
|
||||
}
|
||||
|
||||
async start(options: CaptureOptions): Promise<void> {
|
||||
if (this.active) this.stop('stopped');
|
||||
this.options = options;
|
||||
this.chunks = [];
|
||||
this.totalSamples = 0;
|
||||
this.vad = options.mode === 'auto' ? new EnergyVad(options.vad ?? DEFAULT_VAD) : null;
|
||||
|
||||
if (!navigator.mediaDevices?.getUserMedia) {
|
||||
throw new Error("Le microphone n'est pas accessible dans cet environnement (getUserMedia indisponible).");
|
||||
}
|
||||
|
||||
const constraints: MediaStreamConstraints = {
|
||||
audio: {
|
||||
deviceId: options.deviceId ? { exact: options.deviceId } : undefined,
|
||||
echoCancellation: true,
|
||||
noiseSuppression: true,
|
||||
autoGainControl: true,
|
||||
channelCount: 1
|
||||
}
|
||||
};
|
||||
try {
|
||||
this.stream = await navigator.mediaDevices.getUserMedia(constraints);
|
||||
} catch (err) {
|
||||
const e = err as DOMException;
|
||||
const detail =
|
||||
e.name === 'NotAllowedError'
|
||||
? "Accès au micro refusé. Vérifiez les paramètres de confidentialité Windows (Microphone) et relancez EveFlow."
|
||||
: e.name === 'NotFoundError'
|
||||
? 'Aucun microphone détecté.'
|
||||
: e.name === 'NotReadableError'
|
||||
? 'Le microphone est utilisé par une autre application.'
|
||||
: `${e.name}: ${e.message}`;
|
||||
Log.error('mic', 'getUserMedia failed', { name: e.name, message: e.message });
|
||||
throw new Error(detail);
|
||||
}
|
||||
|
||||
try {
|
||||
this.ctx = new AudioContext({ sampleRate: 16_000, latencyHint: 'interactive' });
|
||||
} catch {
|
||||
this.ctx = new AudioContext({ latencyHint: 'interactive' });
|
||||
}
|
||||
if (this.ctx.state === 'suspended') await this.ctx.resume().catch(() => undefined);
|
||||
this.sampleRate = this.ctx.sampleRate;
|
||||
|
||||
const source = this.ctx.createMediaStreamSource(this.stream);
|
||||
const analyser = this.ctx.createAnalyser();
|
||||
analyser.fftSize = 512;
|
||||
analyser.smoothingTimeConstant = 0.5;
|
||||
source.connect(analyser);
|
||||
audioBus.setInputAnalyser(analyser);
|
||||
|
||||
try {
|
||||
await this.ctx.audioWorklet.addModule(getWorkletUrl());
|
||||
this.worklet = new AudioWorkletNode(this.ctx, 'eveflow-capture', { numberOfInputs: 1, numberOfOutputs: 0, channelCount: 1 });
|
||||
this.worklet.port.onmessage = (event: MessageEvent<Float32Array>) => this.onSamples(event.data);
|
||||
source.connect(this.worklet);
|
||||
} catch (err) {
|
||||
Log.warn('mic', `AudioWorklet unavailable, falling back to ScriptProcessor: ${(err as Error).message}`);
|
||||
const processor = this.ctx.createScriptProcessor(2048, 1, 1);
|
||||
processor.onaudioprocess = (e) => this.onSamples(new Float32Array(e.inputBuffer.getChannelData(0)));
|
||||
const mute = this.ctx.createGain();
|
||||
mute.gain.value = 0; // Chrome requires a destination path, but we never want to hear the mic.
|
||||
source.connect(processor);
|
||||
processor.connect(mute);
|
||||
mute.connect(this.ctx.destination);
|
||||
this.legacy = processor;
|
||||
}
|
||||
|
||||
this.active = true;
|
||||
Log.info('mic', `capture started (${this.sampleRate} Hz, mode=${options.mode})`);
|
||||
}
|
||||
|
||||
private onSamples(samples: Float32Array): void {
|
||||
if (!this.active || !this.options) return;
|
||||
this.chunks.push(samples);
|
||||
this.totalSamples += samples.length;
|
||||
const level = rms(samples);
|
||||
this.options.callbacks.onLevel?.(Math.min(1, level * 6));
|
||||
|
||||
if (this.vad) {
|
||||
const frameMs = (samples.length / this.sampleRate) * 1000;
|
||||
const signal = this.vad.feed(level, frameMs);
|
||||
this.handleVad(signal);
|
||||
} else if (this.totalSamples / this.sampleRate > 120) {
|
||||
this.finishUtterance('max-length');
|
||||
}
|
||||
}
|
||||
|
||||
private handleVad(signal: VadSignal): void {
|
||||
switch (signal) {
|
||||
case 'speech-start':
|
||||
this.options?.callbacks.onSpeechStart?.();
|
||||
break;
|
||||
case 'speech-end':
|
||||
case 'max-length':
|
||||
this.finishUtterance(signal);
|
||||
break;
|
||||
case 'too-short':
|
||||
case 'no-speech':
|
||||
this.stop(signal);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private finishUtterance(reason: 'speech-end' | 'max-length' | 'stopped'): void {
|
||||
const opts = this.options;
|
||||
const chunks = this.chunks;
|
||||
const rate = this.sampleRate;
|
||||
this.teardown();
|
||||
if (!opts) return;
|
||||
if (chunks.length === 0) {
|
||||
opts.callbacks.onEnd('no-speech');
|
||||
return;
|
||||
}
|
||||
const wav = buildWav16k(chunks, rate);
|
||||
Log.info('mic', `utterance captured: ${wav.durationSec.toFixed(2)}s, ${wav.bytes.length} bytes`);
|
||||
if (wav.durationSec < 0.25) {
|
||||
opts.callbacks.onEnd('too-short');
|
||||
return;
|
||||
}
|
||||
opts.callbacks.onUtterance(wav);
|
||||
opts.callbacks.onEnd(reason);
|
||||
}
|
||||
|
||||
/** Stop capture. In manual mode this finalises the utterance; otherwise reports the reason. */
|
||||
stop(reason: 'stopped' | 'no-speech' | 'too-short' = 'stopped'): void {
|
||||
if (!this.active) return;
|
||||
if (reason === 'stopped') {
|
||||
this.finishUtterance('stopped');
|
||||
return;
|
||||
}
|
||||
const opts = this.options;
|
||||
this.teardown();
|
||||
opts?.callbacks.onEnd(reason);
|
||||
}
|
||||
|
||||
/** Abort without producing an utterance. */
|
||||
cancel(): void {
|
||||
if (!this.active) return;
|
||||
this.teardown();
|
||||
this.options?.callbacks.onEnd('stopped');
|
||||
}
|
||||
|
||||
private teardown(): void {
|
||||
this.active = false;
|
||||
audioBus.setInputAnalyser(null);
|
||||
if (this.worklet) {
|
||||
this.worklet.port.onmessage = null;
|
||||
this.worklet.disconnect();
|
||||
this.worklet = null;
|
||||
}
|
||||
if (this.legacy) {
|
||||
this.legacy.onaudioprocess = null;
|
||||
this.legacy.disconnect();
|
||||
this.legacy = null;
|
||||
}
|
||||
this.stream?.getTracks().forEach((t) => t.stop());
|
||||
this.stream = null;
|
||||
void this.ctx?.close().catch(() => undefined);
|
||||
this.ctx = null;
|
||||
this.chunks = [];
|
||||
this.totalSamples = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Singleton facade over the TTS engine bound to the settings store, exposing speaking state
|
||||
* to the voice and chat stores.
|
||||
*/
|
||||
import { useChat } from '../../state/chat';
|
||||
import { useSettings } from '../../state/settings';
|
||||
import { useVoice } from '../../state/voice';
|
||||
import { TtsEngine } from './tts';
|
||||
|
||||
class SpeechFacade {
|
||||
private engine: TtsEngine | null = null;
|
||||
private streaming = false;
|
||||
private streamEnabled = true;
|
||||
|
||||
private get tts(): TtsEngine {
|
||||
if (!this.engine) {
|
||||
const settings = useSettings.getState().settings;
|
||||
this.engine = new TtsEngine(settings.speech);
|
||||
this.engine.onState((state) => {
|
||||
useVoice.getState().setTts(state);
|
||||
const chat = useChat.getState();
|
||||
if (state === 'speaking') chat.setHud('speaking');
|
||||
else if (state === 'idle' && chat.hud === 'speaking') chat.setHud(chat.isSending ? 'thinking' : 'idle');
|
||||
});
|
||||
useSettings.subscribe((s) => this.engine?.updateConfig(s.settings.speech));
|
||||
}
|
||||
return this.engine;
|
||||
}
|
||||
|
||||
init(): void {
|
||||
this.tts;
|
||||
}
|
||||
|
||||
isSpeaking(): boolean {
|
||||
return this.engine?.isActive ?? false;
|
||||
}
|
||||
|
||||
say(text: string): void {
|
||||
if (useSettings.getState().settings.speech.provider === 'off') return;
|
||||
this.tts.speak(text);
|
||||
}
|
||||
|
||||
pushStream(delta: string): void {
|
||||
if (!useSettings.getState().settings.speech.autoSpeak) return;
|
||||
this.streaming = true;
|
||||
if (this.streamEnabled) this.tts.pushStream(delta);
|
||||
}
|
||||
|
||||
endStream(): void {
|
||||
if (this.streaming) this.tts.endStream();
|
||||
this.streaming = false;
|
||||
}
|
||||
|
||||
discardStream(): void {
|
||||
this.streaming = false;
|
||||
this.tts.stop();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.streaming = false;
|
||||
this.engine?.stop();
|
||||
}
|
||||
}
|
||||
|
||||
export const speech = new SpeechFacade();
|
||||
@@ -0,0 +1,110 @@
|
||||
import { Log } from '../../lib/log';
|
||||
import { httpFetch } from '../../lib/transport';
|
||||
|
||||
export type SttProvider = 'openai-compatible' | 'browser';
|
||||
|
||||
export interface SttConfig {
|
||||
provider: SttProvider;
|
||||
apiUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
language: string; // BCP-47, e.g. fr-FR
|
||||
prompt?: string;
|
||||
}
|
||||
|
||||
export function sttEndpoint(apiUrl: string): string {
|
||||
const base = apiUrl.trim().replace(/\/+$/, '');
|
||||
if (base.endsWith('/transcriptions')) return base;
|
||||
if (base.endsWith('/audio')) return `${base}/transcriptions`;
|
||||
if (/\/v\d+$/.test(base)) return `${base}/audio/transcriptions`;
|
||||
return `${base}/v1/audio/transcriptions`;
|
||||
}
|
||||
|
||||
/** Transcribe a WAV buffer through any OpenAI-compatible `/v1/audio/transcriptions` endpoint. */
|
||||
export async function transcribeWav(wav: Uint8Array, config: SttConfig): Promise<string> {
|
||||
if (!config.apiUrl.trim()) throw new Error("URL de l'API de transcription non configurée.");
|
||||
const endpoint = sttEndpoint(config.apiUrl);
|
||||
const fields: Record<string, string> = {
|
||||
model: config.model || 'whisper-1',
|
||||
response_format: 'json'
|
||||
};
|
||||
if (config.language) fields.language = config.language.split('-')[0];
|
||||
if (config.prompt) fields.prompt = config.prompt;
|
||||
const headers: Record<string, string> = {};
|
||||
if (config.apiKey.trim()) headers.Authorization = `Bearer ${config.apiKey.trim()}`;
|
||||
|
||||
Log.info('stt', `POST ${endpoint}`, { model: fields.model, bytes: wav.length });
|
||||
const res = await httpFetch({
|
||||
url: endpoint,
|
||||
method: 'POST',
|
||||
headers,
|
||||
multipart: { fields, file: { name: 'audio.wav', type: 'audio/wav', data: wav } },
|
||||
timeoutMs: 90_000
|
||||
});
|
||||
if (!res.ok) {
|
||||
let detail = res.text ?? '';
|
||||
try {
|
||||
const parsed = JSON.parse(detail) as { error?: { message?: string }; message?: string; detail?: string };
|
||||
detail = parsed.error?.message ?? parsed.message ?? parsed.detail ?? detail;
|
||||
} catch {
|
||||
/* raw text */
|
||||
}
|
||||
throw new Error(`Transcription refusée (HTTP ${res.status}) : ${detail.slice(0, 240)}`);
|
||||
}
|
||||
const text = res.text ?? '';
|
||||
try {
|
||||
const parsed = JSON.parse(text) as { text?: string; transcript?: string; segments?: Array<{ text: string }> };
|
||||
const value = parsed.text ?? parsed.transcript ?? parsed.segments?.map((s) => s.text).join(' ');
|
||||
if (typeof value === 'string') return value.trim();
|
||||
} catch {
|
||||
if (text.trim()) return text.trim();
|
||||
}
|
||||
throw new Error('Réponse de transcription invalide.');
|
||||
}
|
||||
|
||||
/** Web Speech API fallback (Chromium routes it to Google; works only online, quality varies). */
|
||||
export class BrowserRecognizer {
|
||||
private recognition: SpeechRecognition | null = null;
|
||||
|
||||
static isSupported(): boolean {
|
||||
return typeof window !== 'undefined' && !!(window.SpeechRecognition || window.webkitSpeechRecognition);
|
||||
}
|
||||
|
||||
start(lang: string, onResult: (text: string) => void, onError: (message: string) => void, onEnd: () => void): void {
|
||||
const Ctor = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||
if (!Ctor) {
|
||||
onError("La reconnaissance vocale du navigateur n'est pas disponible.");
|
||||
return;
|
||||
}
|
||||
this.stop();
|
||||
const rec = new Ctor();
|
||||
rec.lang = lang;
|
||||
rec.continuous = false;
|
||||
rec.interimResults = false;
|
||||
rec.maxAlternatives = 1;
|
||||
rec.onresult = (event) => {
|
||||
const text = event.results[0]?.[0]?.transcript ?? '';
|
||||
onResult(text);
|
||||
};
|
||||
rec.onerror = (event) => onError(event.error === 'not-allowed' ? 'Micro refusé par le navigateur.' : `Reconnaissance: ${event.error}`);
|
||||
rec.onend = () => {
|
||||
this.recognition = null;
|
||||
onEnd();
|
||||
};
|
||||
this.recognition = rec;
|
||||
try {
|
||||
rec.start();
|
||||
} catch (err) {
|
||||
onError((err as Error).message);
|
||||
}
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
try {
|
||||
this.recognition?.stop();
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
this.recognition = null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,287 @@
|
||||
/**
|
||||
* Text-to-speech engine with a sentence queue, prefetching and Web Audio playback so the HUD
|
||||
* reacts to the actual waveform. Providers: OpenAI-compatible /v1/audio/speech, system voices,
|
||||
* and the legacy Google Translate endpoint (no key, online only).
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { chunkForSpeech, cleanForSpeech, extractSentences } from '../../lib/text';
|
||||
import { httpFetch } from '../../lib/transport';
|
||||
import { audioBus } from './audioBus';
|
||||
|
||||
export type TtsProvider = 'openai-compatible' | 'system' | 'google-free' | 'off';
|
||||
|
||||
export interface TtsConfig {
|
||||
provider: TtsProvider;
|
||||
apiUrl: string;
|
||||
apiKey: string;
|
||||
model: string;
|
||||
voice: string;
|
||||
speed: number; // 0.5 - 2
|
||||
format: 'mp3' | 'wav' | 'opus';
|
||||
systemVoice: string;
|
||||
language: string;
|
||||
volume: number;
|
||||
}
|
||||
|
||||
export type TtsState = 'idle' | 'loading' | 'speaking';
|
||||
|
||||
interface QueueItem {
|
||||
text: string;
|
||||
audio?: Promise<AudioBuffer | null>;
|
||||
}
|
||||
|
||||
export function ttsEndpoint(apiUrl: string): string {
|
||||
const base = apiUrl.trim().replace(/\/+$/, '');
|
||||
if (base.endsWith('/speech')) return base;
|
||||
if (base.endsWith('/audio')) return `${base}/speech`;
|
||||
if (/\/v\d+$/.test(base)) return `${base}/audio/speech`;
|
||||
return `${base}/v1/audio/speech`;
|
||||
}
|
||||
|
||||
export class TtsEngine {
|
||||
private queue: QueueItem[] = [];
|
||||
private playing = false;
|
||||
private currentSource: AudioBufferSourceNode | null = null;
|
||||
private currentUtterance: SpeechSynthesisUtterance | null = null;
|
||||
private streamBuffer = '';
|
||||
private generation = 0;
|
||||
private config: TtsConfig;
|
||||
private listeners = new Set<(state: TtsState) => void>();
|
||||
private state: TtsState = 'idle';
|
||||
|
||||
constructor(config: TtsConfig) {
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
updateConfig(config: TtsConfig): void {
|
||||
this.config = config;
|
||||
audioBus.setVolume(config.volume);
|
||||
}
|
||||
|
||||
onState(listener: (state: TtsState) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
get isActive(): boolean {
|
||||
return this.playing || this.queue.length > 0;
|
||||
}
|
||||
|
||||
private setState(state: TtsState): void {
|
||||
if (this.state === state) return;
|
||||
this.state = state;
|
||||
for (const l of this.listeners) l(state);
|
||||
}
|
||||
|
||||
/** Speak a finished text (clears any pending speech first). */
|
||||
speak(text: string): void {
|
||||
this.stop();
|
||||
for (const chunk of chunkForSpeech(text)) this.enqueue(chunk);
|
||||
}
|
||||
|
||||
/** Feed streamed tokens; complete sentences are spoken as soon as they are available. */
|
||||
pushStream(delta: string): void {
|
||||
this.streamBuffer += delta;
|
||||
const { sentences, rest } = extractSentences(this.streamBuffer);
|
||||
this.streamBuffer = rest;
|
||||
for (const sentence of sentences) this.enqueue(sentence);
|
||||
}
|
||||
|
||||
/** Flush the remainder of a streamed message. */
|
||||
endStream(): void {
|
||||
const rest = this.streamBuffer.trim();
|
||||
this.streamBuffer = '';
|
||||
if (rest) for (const chunk of chunkForSpeech(rest)) this.enqueue(chunk);
|
||||
}
|
||||
|
||||
enqueue(text: string): void {
|
||||
if (this.config.provider === 'off') return;
|
||||
const clean = cleanForSpeech(text);
|
||||
if (!clean || !/[\p{L}\p{N}]/u.test(clean)) return;
|
||||
const item: QueueItem = { text: clean };
|
||||
if (this.config.provider !== 'system') item.audio = this.prefetch(clean, this.generation);
|
||||
this.queue.push(item);
|
||||
void this.drain();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.generation++;
|
||||
this.queue = [];
|
||||
this.streamBuffer = '';
|
||||
if (this.currentSource) {
|
||||
try {
|
||||
this.currentSource.onended = null;
|
||||
this.currentSource.stop();
|
||||
} catch {
|
||||
/* already stopped */
|
||||
}
|
||||
this.currentSource = null;
|
||||
}
|
||||
if (this.currentUtterance) {
|
||||
this.currentUtterance.onend = null;
|
||||
this.currentUtterance = null;
|
||||
}
|
||||
if (typeof speechSynthesis !== 'undefined') speechSynthesis.cancel();
|
||||
audioBus.setSyntheticLevel(0);
|
||||
this.playing = false;
|
||||
this.setState('idle');
|
||||
}
|
||||
|
||||
private async drain(): Promise<void> {
|
||||
if (this.playing) return;
|
||||
this.playing = true;
|
||||
const gen = this.generation;
|
||||
while (this.queue.length > 0 && gen === this.generation) {
|
||||
const item = this.queue.shift()!;
|
||||
try {
|
||||
if (item.audio) {
|
||||
this.setState('loading');
|
||||
const buffer = await item.audio;
|
||||
if (gen !== this.generation) break;
|
||||
if (buffer) {
|
||||
this.setState('speaking');
|
||||
await this.playBuffer(buffer, gen);
|
||||
} else {
|
||||
await this.speakSystem(item.text, gen);
|
||||
}
|
||||
} else {
|
||||
this.setState('speaking');
|
||||
await this.speakSystem(item.text, gen);
|
||||
}
|
||||
} catch (err) {
|
||||
Log.warn('tts', `segment failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
if (gen === this.generation) {
|
||||
this.playing = false;
|
||||
this.setState('idle');
|
||||
}
|
||||
}
|
||||
|
||||
private prefetch(text: string, gen: number): Promise<AudioBuffer | null> {
|
||||
const task = this.config.provider === 'google-free' ? this.fetchGoogle(text) : this.fetchOpenAi(text);
|
||||
return task
|
||||
.then(async (bytes) => {
|
||||
if (gen !== this.generation || !bytes) return null;
|
||||
await audioBus.resume();
|
||||
const copy = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength) as ArrayBuffer;
|
||||
return audioBus.context.decodeAudioData(copy);
|
||||
})
|
||||
.catch((err) => {
|
||||
Log.warn('tts', `synthesis failed, falling back to system voice: ${(err as Error).message}`);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
private async fetchOpenAi(text: string): Promise<Uint8Array> {
|
||||
const { apiUrl, apiKey, model, voice, speed, format } = this.config;
|
||||
if (!apiUrl.trim()) throw new Error('URL TTS non configurée');
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (apiKey.trim()) headers.Authorization = `Bearer ${apiKey.trim()}`;
|
||||
const res = await httpFetch({
|
||||
url: ttsEndpoint(apiUrl),
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({
|
||||
model: model || 'tts-1',
|
||||
input: text,
|
||||
voice: voice || 'alloy',
|
||||
speed: Math.max(0.5, Math.min(2, speed || 1)),
|
||||
response_format: format || 'mp3'
|
||||
}),
|
||||
responseType: 'binary',
|
||||
timeoutMs: 60_000
|
||||
});
|
||||
if (!res.ok || !res.binary) {
|
||||
const body = res.binary ? new TextDecoder().decode(res.binary).slice(0, 200) : '';
|
||||
throw new Error(`HTTP ${res.status} ${body}`);
|
||||
}
|
||||
return res.binary;
|
||||
}
|
||||
|
||||
private async fetchGoogle(text: string): Promise<Uint8Array> {
|
||||
const url = `https://translate.google.com/translate_tts?ie=UTF-8&tl=${encodeURIComponent(this.config.language.split('-')[0] || 'fr')}&client=tw-ob&q=${encodeURIComponent(text.slice(0, 200))}`;
|
||||
const res = await httpFetch({
|
||||
url,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36',
|
||||
Referer: 'https://translate.google.com/'
|
||||
},
|
||||
responseType: 'binary',
|
||||
timeoutMs: 15_000
|
||||
});
|
||||
if (!res.ok || !res.binary) throw new Error(`Google TTS HTTP ${res.status}`);
|
||||
return res.binary;
|
||||
}
|
||||
|
||||
private playBuffer(buffer: AudioBuffer, gen: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (gen !== this.generation) return resolve();
|
||||
const ctx = audioBus.context;
|
||||
const source = ctx.createBufferSource();
|
||||
source.buffer = buffer;
|
||||
if (this.config.provider === 'google-free') source.playbackRate.value = Math.max(0.5, Math.min(2, this.config.speed || 1));
|
||||
source.connect(audioBus.output);
|
||||
source.onended = () => {
|
||||
if (this.currentSource === source) this.currentSource = null;
|
||||
resolve();
|
||||
};
|
||||
this.currentSource = source;
|
||||
source.start();
|
||||
});
|
||||
}
|
||||
|
||||
private speakSystem(text: string, gen: number): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
if (typeof speechSynthesis === 'undefined' || gen !== this.generation) return resolve();
|
||||
const utterance = new SpeechSynthesisUtterance(text);
|
||||
utterance.lang = this.config.language || 'fr-FR';
|
||||
utterance.rate = Math.max(0.5, Math.min(2, this.config.speed || 1));
|
||||
utterance.pitch = 1;
|
||||
utterance.volume = Math.max(0, Math.min(1, this.config.volume));
|
||||
const voice = this.resolveSystemVoice();
|
||||
if (voice) utterance.voice = voice;
|
||||
let pulse: ReturnType<typeof setInterval> | null = setInterval(() => {
|
||||
audioBus.setSyntheticLevel(0.35 + Math.random() * 0.4);
|
||||
}, 90);
|
||||
const finish = () => {
|
||||
if (pulse) clearInterval(pulse);
|
||||
pulse = null;
|
||||
audioBus.setSyntheticLevel(0);
|
||||
if (this.currentUtterance === utterance) this.currentUtterance = null;
|
||||
resolve();
|
||||
};
|
||||
utterance.onend = finish;
|
||||
utterance.onerror = finish;
|
||||
this.currentUtterance = utterance;
|
||||
speechSynthesis.speak(utterance);
|
||||
});
|
||||
}
|
||||
|
||||
private resolveSystemVoice(): SpeechSynthesisVoice | undefined {
|
||||
if (typeof speechSynthesis === 'undefined') return undefined;
|
||||
const voices = speechSynthesis.getVoices();
|
||||
if (this.config.systemVoice) {
|
||||
const exact = voices.find((v) => v.name === this.config.systemVoice);
|
||||
if (exact) return exact;
|
||||
}
|
||||
const lang = (this.config.language || 'fr').toLowerCase().split('-')[0];
|
||||
const candidates = voices.filter((v) => v.lang.toLowerCase().startsWith(lang));
|
||||
return candidates.sort((a, b) => scoreVoice(b) - scoreVoice(a))[0] ?? voices[0];
|
||||
}
|
||||
}
|
||||
|
||||
export function scoreVoice(v: SpeechSynthesisVoice): number {
|
||||
const name = v.name.toLowerCase();
|
||||
let score = 0;
|
||||
if (name.includes('natural') || name.includes('neural') || name.includes('online')) score += 40;
|
||||
if (name.includes('google')) score += 30;
|
||||
if (name.includes('microsoft')) score += 10;
|
||||
if (v.localService) score += 5;
|
||||
return score;
|
||||
}
|
||||
|
||||
export function listSystemVoices(): SpeechSynthesisVoice[] {
|
||||
if (typeof speechSynthesis === 'undefined') return [];
|
||||
return [...speechSynthesis.getVoices()].sort((a, b) => a.lang.localeCompare(b.lang) || scoreVoice(b) - scoreVoice(a));
|
||||
}
|
||||
@@ -0,0 +1,107 @@
|
||||
/**
|
||||
* Energy-based voice activity detector with an adaptive noise floor.
|
||||
* Pure logic (no Web Audio) so it can be unit tested.
|
||||
*/
|
||||
export interface VadOptions {
|
||||
/** Signal must exceed noiseFloor * ratio (and absolute minimum) to count as speech. */
|
||||
speechRatio: number;
|
||||
minRms: number;
|
||||
/** Silence duration that ends an utterance. */
|
||||
silenceMs: number;
|
||||
/** Minimum duration of speech before it is considered a real utterance. */
|
||||
minSpeechMs: number;
|
||||
/** Hard cap on one utterance. */
|
||||
maxUtteranceMs: number;
|
||||
/** Give up if nobody speaks for this long after starting. */
|
||||
noSpeechTimeoutMs: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_VAD: VadOptions = {
|
||||
speechRatio: 2.6,
|
||||
minRms: 0.012,
|
||||
silenceMs: 900,
|
||||
minSpeechMs: 250,
|
||||
maxUtteranceMs: 45_000,
|
||||
noSpeechTimeoutMs: 9_000
|
||||
};
|
||||
|
||||
export type VadSignal = 'idle' | 'speech-start' | 'speaking' | 'speech-end' | 'too-short' | 'no-speech' | 'max-length';
|
||||
|
||||
export class EnergyVad {
|
||||
private noiseFloor: number;
|
||||
private speaking = false;
|
||||
private speechMs = 0;
|
||||
private silenceMs = 0;
|
||||
private elapsedMs = 0;
|
||||
private calibrationMs = 0;
|
||||
readonly options: VadOptions;
|
||||
|
||||
constructor(options: Partial<VadOptions> = {}) {
|
||||
this.options = { ...DEFAULT_VAD, ...options };
|
||||
this.noiseFloor = this.options.minRms;
|
||||
}
|
||||
|
||||
get isSpeaking(): boolean {
|
||||
return this.speaking;
|
||||
}
|
||||
|
||||
get floor(): number {
|
||||
return this.noiseFloor;
|
||||
}
|
||||
|
||||
/** Feed one analysis frame. Returns the transition that occurred. */
|
||||
feed(frameRms: number, frameMs: number): VadSignal {
|
||||
const o = this.options;
|
||||
this.elapsedMs += frameMs;
|
||||
const threshold = Math.max(o.minRms, this.noiseFloor * o.speechRatio);
|
||||
const loud = frameRms > threshold;
|
||||
|
||||
// Adaptive noise floor: fast during the first 400 ms, slow afterwards, only while not speaking.
|
||||
if (!loud || this.calibrationMs < 400) {
|
||||
const alpha = this.calibrationMs < 400 ? 0.25 : 0.02;
|
||||
this.noiseFloor = this.noiseFloor * (1 - alpha) + frameRms * alpha;
|
||||
this.calibrationMs += frameMs;
|
||||
}
|
||||
|
||||
if (!this.speaking) {
|
||||
if (loud) {
|
||||
this.speechMs += frameMs;
|
||||
if (this.speechMs >= Math.min(120, o.minSpeechMs)) {
|
||||
this.speaking = true;
|
||||
this.silenceMs = 0;
|
||||
return 'speech-start';
|
||||
}
|
||||
} else {
|
||||
this.speechMs = 0;
|
||||
if (this.elapsedMs >= o.noSpeechTimeoutMs) return 'no-speech';
|
||||
}
|
||||
return 'idle';
|
||||
}
|
||||
|
||||
// speaking
|
||||
this.speechMs += frameMs;
|
||||
if (loud) this.silenceMs = 0;
|
||||
else this.silenceMs += frameMs;
|
||||
|
||||
if (this.speechMs >= o.maxUtteranceMs) return this.finish('max-length');
|
||||
if (this.silenceMs >= o.silenceMs) {
|
||||
const spoken = this.speechMs - this.silenceMs;
|
||||
return this.finish(spoken >= o.minSpeechMs ? 'speech-end' : 'too-short');
|
||||
}
|
||||
return 'speaking';
|
||||
}
|
||||
|
||||
private finish(signal: VadSignal): VadSignal {
|
||||
this.speaking = false;
|
||||
this.speechMs = 0;
|
||||
this.silenceMs = 0;
|
||||
return signal;
|
||||
}
|
||||
|
||||
reset(): void {
|
||||
this.speaking = false;
|
||||
this.speechMs = 0;
|
||||
this.silenceMs = 0;
|
||||
this.elapsedMs = 0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
/**
|
||||
* Voice controller: microphone → VAD → STT → conversation, plus hands-free loop and barge-in.
|
||||
*/
|
||||
import { Log } from '../../lib/log';
|
||||
import { useChat } from '../../state/chat';
|
||||
import { useSettings } from '../../state/settings';
|
||||
import { useVoice } from '../../state/voice';
|
||||
import { sendMessage } from '../conversation';
|
||||
import { audioBus } from './audioBus';
|
||||
import { listMicrophones, MicCapture } from './capture';
|
||||
import { speech } from './speech';
|
||||
import { BrowserRecognizer, transcribeWav } from './stt';
|
||||
import type { WavResult } from './wav';
|
||||
|
||||
const SENSITIVITY_RATIO: Record<number, number> = { 1: 4.5, 2: 3.4, 3: 2.6, 4: 2.0, 5: 1.6 };
|
||||
const SENSITIVITY_MIN_RMS: Record<number, number> = { 1: 0.03, 2: 0.02, 3: 0.012, 4: 0.008, 5: 0.005 };
|
||||
|
||||
class VoiceController {
|
||||
private capture = new MicCapture();
|
||||
private browser = new BrowserRecognizer();
|
||||
private handsFreeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private stopping = false;
|
||||
private unsubscribeVoice: (() => void) | null = null;
|
||||
|
||||
init(): void {
|
||||
void listMicrophones().then((devices) => useVoice.getState().setMicDevices(devices));
|
||||
// Hands-free: re-arm the microphone once the assistant stops speaking.
|
||||
this.unsubscribeVoice = useVoice.subscribe((state, prev) => {
|
||||
if (prev.tts !== 'idle' && state.tts === 'idle' && state.handsFree && state.phase === 'off') {
|
||||
this.scheduleHandsFree(350);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
dispose(): void {
|
||||
this.unsubscribeVoice?.();
|
||||
this.stop();
|
||||
}
|
||||
|
||||
get isListening(): boolean {
|
||||
return useVoice.getState().phase !== 'off';
|
||||
}
|
||||
|
||||
toggle(): void {
|
||||
if (this.isListening) this.stop();
|
||||
else void this.start();
|
||||
}
|
||||
|
||||
setHandsFree(on: boolean): void {
|
||||
useVoice.getState().setHandsFree(on);
|
||||
useSettings.getState().update({ voice: { handsFree: on } });
|
||||
if (on && !this.isListening && !speech.isSpeaking()) void this.start();
|
||||
if (!on) {
|
||||
this.clearHandsFreeTimer();
|
||||
}
|
||||
}
|
||||
|
||||
private scheduleHandsFree(delay: number): void {
|
||||
this.clearHandsFreeTimer();
|
||||
this.handsFreeTimer = setTimeout(() => {
|
||||
const v = useVoice.getState();
|
||||
if (v.handsFree && v.phase === 'off' && !useChat.getState().isSending && !speech.isSpeaking()) void this.start();
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private clearHandsFreeTimer(): void {
|
||||
if (this.handsFreeTimer) clearTimeout(this.handsFreeTimer);
|
||||
this.handsFreeTimer = null;
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
const voice = useVoice.getState();
|
||||
if (voice.phase !== 'off') return;
|
||||
const settings = useSettings.getState().settings.voice;
|
||||
voice.setError(null);
|
||||
voice.setInterim('');
|
||||
if (!settings.bargeIn) speech.stop();
|
||||
else speech.stop();
|
||||
useChat.getState().setHud('listening');
|
||||
voice.setPhase('arming');
|
||||
|
||||
if (settings.provider === 'browser') {
|
||||
this.startBrowser(settings.language);
|
||||
return;
|
||||
}
|
||||
|
||||
const sensitivity = Math.min(5, Math.max(1, Math.round(settings.sensitivity))) as 1 | 2 | 3 | 4 | 5;
|
||||
try {
|
||||
await audioBus.resume();
|
||||
await this.capture.start({
|
||||
deviceId: settings.micDeviceId || undefined,
|
||||
mode: settings.captureMode,
|
||||
vad: {
|
||||
silenceMs: settings.silenceMs,
|
||||
speechRatio: SENSITIVITY_RATIO[sensitivity],
|
||||
minRms: SENSITIVITY_MIN_RMS[sensitivity],
|
||||
noSpeechTimeoutMs: useVoice.getState().handsFree ? 20_000 : 9_000
|
||||
},
|
||||
callbacks: {
|
||||
onLevel: (level) => useVoice.getState().setInputLevel(level),
|
||||
onSpeechStart: () => useVoice.getState().setPhase('speech'),
|
||||
onUtterance: (wav) => void this.transcribe(wav),
|
||||
onEnd: (reason) => this.onCaptureEnd(reason),
|
||||
onError: (message) => useVoice.getState().setError(message)
|
||||
}
|
||||
});
|
||||
useVoice.getState().setPhase('listening');
|
||||
this.playChime(true);
|
||||
} catch (err) {
|
||||
const message = (err as Error).message;
|
||||
useVoice.getState().setError(message);
|
||||
useVoice.getState().setPhase('off');
|
||||
useChat.getState().setHud('error');
|
||||
useChat.getState().setError(message);
|
||||
Log.error('voice', message);
|
||||
setTimeout(() => useChat.getState().setHud('idle'), 2500);
|
||||
}
|
||||
}
|
||||
|
||||
private startBrowser(lang: string): void {
|
||||
useVoice.getState().setPhase('listening');
|
||||
this.browser.start(
|
||||
lang,
|
||||
(text) => {
|
||||
useVoice.getState().setTranscript(text);
|
||||
if (text.trim()) void sendMessage(text, [], 'voice');
|
||||
},
|
||||
(message) => {
|
||||
useVoice.getState().setError(message);
|
||||
useChat.getState().setError(message);
|
||||
},
|
||||
() => this.onCaptureEnd('stopped')
|
||||
);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopping = true;
|
||||
this.clearHandsFreeTimer();
|
||||
if (useSettings.getState().settings.voice.provider === 'browser') this.browser.stop();
|
||||
else if (this.capture.isActive) this.capture.stop('stopped');
|
||||
else this.onCaptureEnd('stopped');
|
||||
this.stopping = false;
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
this.clearHandsFreeTimer();
|
||||
this.browser.stop();
|
||||
this.capture.cancel();
|
||||
}
|
||||
|
||||
private onCaptureEnd(reason: string): void {
|
||||
const voice = useVoice.getState();
|
||||
voice.setInputLevel(0);
|
||||
if (voice.phase !== 'transcribing') voice.setPhase('off');
|
||||
const chat = useChat.getState();
|
||||
if (chat.hud === 'listening') chat.setHud(chat.isSending ? 'thinking' : 'idle');
|
||||
if (reason === 'no-speech') {
|
||||
if (voice.handsFree && !this.stopping) this.scheduleHandsFree(400);
|
||||
else voice.setError("Aucune parole détectée.");
|
||||
} else if (reason === 'too-short') {
|
||||
if (voice.handsFree && !this.stopping) this.scheduleHandsFree(200);
|
||||
}
|
||||
}
|
||||
|
||||
private async transcribe(wav: WavResult): Promise<void> {
|
||||
const voice = useVoice.getState();
|
||||
voice.setPhase('transcribing');
|
||||
useChat.getState().setHud('thinking');
|
||||
this.playChime(false);
|
||||
const settings = useSettings.getState().settings.voice;
|
||||
try {
|
||||
const text = await transcribeWav(wav.bytes, settings);
|
||||
voice.setTranscript(text);
|
||||
voice.setPhase('off');
|
||||
Log.info('voice', `transcript (${wav.durationSec.toFixed(1)}s): ${text}`);
|
||||
if (text.trim()) {
|
||||
await sendMessage(text, [], 'voice');
|
||||
} else {
|
||||
voice.setError('Transcription vide.');
|
||||
useChat.getState().setHud('idle');
|
||||
if (voice.handsFree) this.scheduleHandsFree(300);
|
||||
}
|
||||
} catch (err) {
|
||||
const message = (err as Error).message;
|
||||
voice.setError(message);
|
||||
voice.setPhase('off');
|
||||
useChat.getState().setError(message);
|
||||
useChat.getState().setHud('error');
|
||||
Log.error('voice', `transcription failed: ${message}`);
|
||||
setTimeout(() => useChat.getState().setHud('idle'), 3000);
|
||||
}
|
||||
}
|
||||
|
||||
private playChime(up: boolean): void {
|
||||
if (!useSettings.getState().settings.voice.wakeChime) return;
|
||||
try {
|
||||
const ctx = audioBus.context;
|
||||
const osc = ctx.createOscillator();
|
||||
const gain = ctx.createGain();
|
||||
osc.type = 'sine';
|
||||
const now = ctx.currentTime;
|
||||
osc.frequency.setValueAtTime(up ? 660 : 880, now);
|
||||
osc.frequency.exponentialRampToValueAtTime(up ? 990 : 550, now + 0.12);
|
||||
gain.gain.setValueAtTime(0.0001, now);
|
||||
gain.gain.exponentialRampToValueAtTime(0.08, now + 0.02);
|
||||
gain.gain.exponentialRampToValueAtTime(0.0001, now + 0.16);
|
||||
osc.connect(gain);
|
||||
gain.connect(audioBus.output);
|
||||
osc.start(now);
|
||||
osc.stop(now + 0.18);
|
||||
} catch {
|
||||
/* audio unavailable */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const voiceController = new VoiceController();
|
||||
@@ -0,0 +1,106 @@
|
||||
/** PCM utilities: merge, resample and encode 16-bit mono WAV. */
|
||||
|
||||
export function mergeChunks(chunks: Float32Array[], totalLength?: number): Float32Array {
|
||||
const length = totalLength ?? chunks.reduce((n, c) => n + c.length, 0);
|
||||
const out = new Float32Array(length);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
if (offset + chunk.length > length) {
|
||||
out.set(chunk.subarray(0, length - offset), offset);
|
||||
break;
|
||||
}
|
||||
out.set(chunk, offset);
|
||||
offset += chunk.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Linear-interpolation resampler with a box pre-filter when downsampling. */
|
||||
export function resample(input: Float32Array, fromRate: number, toRate: number): Float32Array {
|
||||
if (fromRate === toRate || input.length === 0) return input;
|
||||
const ratio = fromRate / toRate;
|
||||
const outLength = Math.max(1, Math.round(input.length / ratio));
|
||||
const out = new Float32Array(outLength);
|
||||
if (ratio > 1) {
|
||||
for (let i = 0; i < outLength; i++) {
|
||||
const start = Math.floor(i * ratio);
|
||||
const end = Math.min(input.length, Math.floor((i + 1) * ratio));
|
||||
let sum = 0;
|
||||
let count = 0;
|
||||
for (let j = start; j < end; j++) {
|
||||
sum += input[j];
|
||||
count++;
|
||||
}
|
||||
out[i] = count > 0 ? sum / count : input[Math.min(start, input.length - 1)];
|
||||
}
|
||||
} else {
|
||||
for (let i = 0; i < outLength; i++) {
|
||||
const pos = i * ratio;
|
||||
const idx = Math.floor(pos);
|
||||
const frac = pos - idx;
|
||||
const a = input[Math.min(idx, input.length - 1)];
|
||||
const b = input[Math.min(idx + 1, input.length - 1)];
|
||||
out[i] = a + (b - a) * frac;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function rms(samples: Float32Array): number {
|
||||
if (samples.length === 0) return 0;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < samples.length; i++) sum += samples[i] * samples[i];
|
||||
return Math.sqrt(sum / samples.length);
|
||||
}
|
||||
|
||||
/** Peak-normalise quiet recordings so ASR models get a healthy signal. */
|
||||
export function normalizeGain(samples: Float32Array, targetPeak = 0.9): Float32Array {
|
||||
let peak = 0;
|
||||
for (let i = 0; i < samples.length; i++) peak = Math.max(peak, Math.abs(samples[i]));
|
||||
if (peak === 0 || peak >= targetPeak * 0.8) return samples;
|
||||
const gain = Math.min(8, targetPeak / peak);
|
||||
const out = new Float32Array(samples.length);
|
||||
for (let i = 0; i < samples.length; i++) out[i] = samples[i] * gain;
|
||||
return out;
|
||||
}
|
||||
|
||||
export function encodeWav(samples: Float32Array, sampleRate: number): Uint8Array {
|
||||
const buffer = new ArrayBuffer(44 + samples.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
const writeString = (offset: number, text: string) => {
|
||||
for (let i = 0; i < text.length; i++) view.setUint8(offset + i, text.charCodeAt(i));
|
||||
};
|
||||
writeString(0, 'RIFF');
|
||||
view.setUint32(4, 36 + samples.length * 2, true);
|
||||
writeString(8, 'WAVE');
|
||||
writeString(12, 'fmt ');
|
||||
view.setUint32(16, 16, true);
|
||||
view.setUint16(20, 1, true);
|
||||
view.setUint16(22, 1, true);
|
||||
view.setUint32(24, sampleRate, true);
|
||||
view.setUint32(28, sampleRate * 2, true);
|
||||
view.setUint16(32, 2, true);
|
||||
view.setUint16(34, 16, true);
|
||||
writeString(36, 'data');
|
||||
view.setUint32(40, samples.length * 2, true);
|
||||
let offset = 44;
|
||||
for (let i = 0; i < samples.length; i++, offset += 2) {
|
||||
const s = Math.max(-1, Math.min(1, samples[i]));
|
||||
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
}
|
||||
return new Uint8Array(buffer);
|
||||
}
|
||||
|
||||
export interface WavResult {
|
||||
bytes: Uint8Array;
|
||||
sampleRate: number;
|
||||
durationSec: number;
|
||||
}
|
||||
|
||||
/** Build a 16 kHz mono WAV (the format every OpenAI-compatible ASR accepts) from captured chunks. */
|
||||
export function buildWav16k(chunks: Float32Array[], captureRate: number): WavResult {
|
||||
const merged = mergeChunks(chunks);
|
||||
const resampled = resample(merged, captureRate, 16_000);
|
||||
const normalized = normalizeGain(resampled);
|
||||
return { bytes: encodeWav(normalized, 16_000), sampleRate: 16_000, durationSec: normalized.length / 16_000 };
|
||||
}
|
||||
Reference in new issue
Block a user