mirror of
https://github.com/R0m1k3/EveFlow.git
synced 2026-10-23 15:18:13 +02:00
feat: fin de phrase Silero, vision d'écran et actions locales (v2.3.0) (#13)
Écoute permanente - Silero VAD dans le worker sherpa-onnx (vad.start/audio/stop) ; le renderer envoie des trames de 128 ms pendant la commande et reçoit le segment WAV complet (speech-start / segment). Repli sur le VAD énergétique si le modèle n'est pas installé ou si l'option est désactivée. - Modèle « silero-vad » (0,6 Mo) dans le catalogue, section « Fin de phrase » dans Modèles locaux, réglage dans Paramètres → Micro. Vision d'écran - IPC system:screen-capture (desktopCapturer, JPEG 1600 px) ; bouton dans la barre de commande ; « regarde mon écran… » joint la capture à la question envoyée à Hermes (transport chat completions pour les images). Actions locales - IPC system:action à liste blanche : verrouiller la session, touches média (volume, mute, lecture, piste), ouvrir une application connue ou une URL http(s), presse-papiers, recherche de fichiers dans les dossiers utilisateur. - Routeur d'intentions FR/EN exécuté avant Hermes (« coupe le son », « ouvre Spotify », « verrouille la session »…), résultat affiché et lu ; désactivable. Correctif - Chat completions : les premiers fragments SSE arrivés avant l'événement de démarrage étaient perdus (premier mot manquant) ; ils sont maintenant rejoués. Tests : 26 tests unitaires (intentions locales ajoutées), e2e Electron sous Xvfb avec les vrais modèles (Silero : un segment par phrase, 6,7 s d'audio traités en 190 ms ; capture 109 ko reçue par Hermes ; intention locale traitée sans Hermes). Claude-Session: https://claude.ai/code/session_017Wn5VX9HNbJ7N54hR24u9Y Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
LogiFlowandClaude authored and 
GitHub committed
1 parent
4f6074183f
commit
af8bdc5e9d
24 files changed
+691
-47
No files matched your search
@@ -0,0 +1,184 @@
|
||||
/**
|
||||
* Local "JARVIS" actions: screenshot for vision, and an allow-list of system actions
|
||||
* (lock, open an application or URL, media keys, clipboard, file search).
|
||||
*/
|
||||
import { app, clipboard, desktopCapturer, ipcMain, screen, shell } from 'electron';
|
||||
import { execFile } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { IPC } from '../../shared/ipc';
|
||||
import type { SystemAction, SystemActionResult } from '../../shared/bridge';
|
||||
import { log } from '../logger';
|
||||
|
||||
const MEDIA_KEYS: Record<string, number> = {
|
||||
'volume-up': 175,
|
||||
'volume-down': 174,
|
||||
mute: 173,
|
||||
'play-pause': 179,
|
||||
next: 176,
|
||||
previous: 177
|
||||
};
|
||||
|
||||
/** Applications the assistant may launch by name (Windows aliases + common Linux/macOS names). */
|
||||
const APP_ALIASES: Record<string, string[]> = {
|
||||
'bloc-notes': ['notepad.exe', 'gedit', 'TextEdit'],
|
||||
notepad: ['notepad.exe', 'gedit', 'TextEdit'],
|
||||
calculatrice: ['calc.exe', 'gnome-calculator', 'Calculator'],
|
||||
calc: ['calc.exe', 'gnome-calculator', 'Calculator'],
|
||||
explorateur: ['explorer.exe', 'nautilus', 'Finder'],
|
||||
explorer: ['explorer.exe', 'nautilus', 'Finder'],
|
||||
terminal: ['wt.exe', 'cmd.exe', 'gnome-terminal', 'Terminal'],
|
||||
cmd: ['cmd.exe'],
|
||||
powershell: ['powershell.exe'],
|
||||
paint: ['mspaint.exe'],
|
||||
chrome: ['chrome', 'google-chrome', 'Google Chrome'],
|
||||
edge: ['msedge', 'microsoft-edge', 'Microsoft Edge'],
|
||||
firefox: ['firefox', 'Firefox'],
|
||||
vscode: ['code', 'Visual Studio Code'],
|
||||
code: ['code', 'Visual Studio Code'],
|
||||
spotify: ['spotify', 'Spotify'],
|
||||
discord: ['discord', 'Discord'],
|
||||
steam: ['steam', 'Steam'],
|
||||
word: ['winword', 'Microsoft Word'],
|
||||
excel: ['excel', 'Microsoft Excel'],
|
||||
outlook: ['outlook', 'Microsoft Outlook'],
|
||||
teams: ['ms-teams', 'Microsoft Teams'],
|
||||
'task manager': ['taskmgr.exe'],
|
||||
'gestionnaire des tâches': ['taskmgr.exe'],
|
||||
paramètres: ['ms-settings:'],
|
||||
settings: ['ms-settings:']
|
||||
};
|
||||
|
||||
function run(cmd: string, args: string[], timeoutMs = 8000): Promise<{ code: number; out: string }> {
|
||||
return new Promise((resolve) => {
|
||||
execFile(cmd, args, { timeout: timeoutMs, windowsHide: true }, (err, stdout, stderr) => {
|
||||
resolve({ code: err ? 1 : 0, out: `${stdout}${stderr}`.trim() });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function openApp(name: string): Promise<SystemActionResult> {
|
||||
const key = name.trim().toLowerCase();
|
||||
if (!key || key.length > 40) return { ok: false, message: 'Nom d’application invalide' };
|
||||
const candidates = APP_ALIASES[key] ?? [key.replace(/[^a-z0-9 ._-]/gi, '')];
|
||||
if (process.platform === 'win32') {
|
||||
for (const candidate of candidates) {
|
||||
if (candidate.endsWith(':')) {
|
||||
await shell.openExternal(candidate);
|
||||
return { ok: true, message: `${name} ouvert` };
|
||||
}
|
||||
// `start` resolves App Paths, PATH and Start Menu names.
|
||||
const result = await run('cmd.exe', ['/c', 'start', '', candidate]);
|
||||
if (result.code === 0) return { ok: true, message: `${name} lancé` };
|
||||
}
|
||||
return { ok: false, message: `Impossible de lancer ${name}` };
|
||||
}
|
||||
for (const candidate of candidates) {
|
||||
const result = process.platform === 'darwin' ? await run('open', ['-a', candidate]) : await run('sh', ['-c', `command -v ${JSON.stringify(candidate)} >/dev/null && (nohup ${JSON.stringify(candidate)} >/dev/null 2>&1 &)`]);
|
||||
if (result.code === 0) return { ok: true, message: `${name} lancé` };
|
||||
}
|
||||
return { ok: false, message: `Application introuvable : ${name}` };
|
||||
}
|
||||
|
||||
async function lockSession(): Promise<SystemActionResult> {
|
||||
if (process.platform === 'win32') {
|
||||
const r = await run('rundll32.exe', ['user32.dll,LockWorkStation']);
|
||||
return { ok: r.code === 0, message: r.code === 0 ? 'Session verrouillée' : r.out };
|
||||
}
|
||||
if (process.platform === 'darwin') {
|
||||
const r = await run('osascript', ['-e', 'tell application "System Events" to keystroke "q" using {command down, control down}']);
|
||||
return { ok: r.code === 0, message: r.out };
|
||||
}
|
||||
const r = await run('sh', ['-c', 'loginctl lock-session || xdg-screensaver lock || gnome-screensaver-command -l']);
|
||||
return { ok: r.code === 0, message: r.code === 0 ? 'Session verrouillée' : r.out };
|
||||
}
|
||||
|
||||
async function mediaKey(key: string): Promise<SystemActionResult> {
|
||||
const code = MEDIA_KEYS[key];
|
||||
if (!code) return { ok: false, message: 'Touche inconnue' };
|
||||
if (process.platform === 'win32') {
|
||||
const script = `$s=Add-Type -MemberDefinition '[DllImport("user32.dll")] public static extern void keybd_event(byte b,byte s,uint f,UIntPtr e);' -Name K -Namespace W -PassThru; $s::keybd_event(${code},0,0,[UIntPtr]::Zero); $s::keybd_event(${code},0,2,[UIntPtr]::Zero)`;
|
||||
const r = await run('powershell.exe', ['-NoProfile', '-NonInteractive', '-Command', script]);
|
||||
return { ok: r.code === 0, message: r.code === 0 ? key : r.out };
|
||||
}
|
||||
const xdo: Record<string, string> = { 'volume-up': 'XF86AudioRaiseVolume', 'volume-down': 'XF86AudioLowerVolume', mute: 'XF86AudioMute', 'play-pause': 'XF86AudioPlay', next: 'XF86AudioNext', previous: 'XF86AudioPrev' };
|
||||
const r = await run('sh', ['-c', `xdotool key ${xdo[key]}`]);
|
||||
return { ok: r.code === 0, message: r.code === 0 ? key : 'xdotool indisponible' };
|
||||
}
|
||||
|
||||
async function findFiles(query: string): Promise<SystemActionResult> {
|
||||
const needle = query.trim().toLowerCase();
|
||||
if (needle.length < 2) return { ok: false, message: 'Recherche trop courte' };
|
||||
const roots = ['Documents', 'Desktop', 'Downloads', 'Pictures'].map((d) => path.join(os.homedir(), d));
|
||||
const hits: string[] = [];
|
||||
const deadline = Date.now() + 4000;
|
||||
const walk = async (dir: string, depth: number) => {
|
||||
if (depth > 4 || hits.length >= 25 || Date.now() > deadline) return;
|
||||
let entries: fs.Dirent[] = [];
|
||||
try {
|
||||
entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith('.') || entry.name === 'node_modules') continue;
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.name.toLowerCase().includes(needle)) hits.push(full);
|
||||
if (entry.isDirectory()) await walk(full, depth + 1);
|
||||
if (hits.length >= 25) return;
|
||||
}
|
||||
};
|
||||
for (const root of roots) await walk(root, 0);
|
||||
return { ok: true, message: `${hits.length} résultat(s)`, data: hits };
|
||||
}
|
||||
|
||||
export async function captureScreen(maxWidth = 1600): Promise<string> {
|
||||
const display = screen.getPrimaryDisplay();
|
||||
const scale = Math.min(1, maxWidth / display.size.width);
|
||||
const sources = await desktopCapturer.getSources({
|
||||
types: ['screen'],
|
||||
thumbnailSize: { width: Math.round(display.size.width * scale), height: Math.round(display.size.height * scale) }
|
||||
});
|
||||
const primary = sources.find((s) => s.display_id === String(display.id)) ?? sources[0];
|
||||
if (!primary) throw new Error('Aucun écran capturable');
|
||||
return primary.thumbnail.toJPEG(82).toString('base64').replace(/^/, 'data:image/jpeg;base64,');
|
||||
}
|
||||
|
||||
export async function runSystemAction(action: SystemAction): Promise<SystemActionResult> {
|
||||
switch (action.type) {
|
||||
case 'lock':
|
||||
return lockSession();
|
||||
case 'open-app':
|
||||
return openApp(String(action.name ?? ''));
|
||||
case 'open-url': {
|
||||
const url = String(action.url ?? '');
|
||||
if (!/^https?:\/\//i.test(url) || url.length > 2048) return { ok: false, message: 'URL refusée' };
|
||||
await shell.openExternal(url);
|
||||
return { ok: true, message: 'Ouvert dans le navigateur' };
|
||||
}
|
||||
case 'media':
|
||||
return mediaKey(String(action.key));
|
||||
case 'clipboard-read':
|
||||
return { ok: true, data: (await clipboard.readText()).slice(0, 20_000) };
|
||||
case 'clipboard-write':
|
||||
await clipboard.writeText(String(action.text ?? '').slice(0, 100_000));
|
||||
return { ok: true, message: 'Copié dans le presse-papiers' };
|
||||
case 'find-files':
|
||||
return findFiles(String(action.query ?? ''));
|
||||
default:
|
||||
return { ok: false, message: 'Action inconnue' };
|
||||
}
|
||||
}
|
||||
|
||||
export function registerSystemIpc(): void {
|
||||
ipcMain.handle(IPC.screenCapture, (_e, maxWidth?: number) => captureScreen(Number(maxWidth) || 1600));
|
||||
ipcMain.handle(IPC.systemAction, async (_e, action: SystemAction) => {
|
||||
if (!action || typeof action.type !== 'string') throw new Error('Action invalide');
|
||||
log('INFO', 'system', `action ${action.type}`);
|
||||
const result = await runSystemAction(action);
|
||||
if (!result.ok) log('WARN', 'system', `action ${action.type} failed: ${result.message}`);
|
||||
return result;
|
||||
});
|
||||
void app;
|
||||
}
|
||||
Reference in new issue
Block a user