diff --git a/README.md b/README.md index 508e7a2..1d75b89 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ La version 2 est une réécriture complète : plus de robot 3D, un pipeline voca * **Serveur MCP intégré** : Hermes se connecte à `http://:7842/mcp` et obtient les outils du PC (capture d'écran renvoyée en image, verrouillage, applications, URL, touches média, presse-papiers, recherche de fichiers, voix, notifications, état du HUD, affichage dans le fil). Même port et même secret que le webhook ; en mode chat completions, les mêmes outils sont proposés directement au modèle. * **Heures calmes et priorités** : plage horaire pendant laquelle les messages poussés s'affichent sans être lus ni faire clignoter le noyau (badge « non lus » à la place), thème nuit automatique, mots prioritaires lus quand même, résumé vocal des rapports longs (les premières phrases seulement). * **Mode mission** : un bouton dans la barre de commande bascule sur un second modèle Hermes (plus puissant) pour les tâches longues ; le modèle rapide reste utilisé pour la conversation courante. +* **Résumé vocal des réponses longues** : seules les premières phrases (réglable) et la question finale sont lues, le reste s'affiche dans le fil ; la conversation Hermes est continue d'un message à l'autre (la session créée par Hermes est reprise à chaque run). * **Widget compact « glanceable »** : état (veille, écoute, réflexion, parle), dernière phrase de l'assistant, badge de non-lus, indicateurs heures calmes et mission. * **Voix Microsoft Edge** (moteur par défaut) : les voix neuronales de la lecture à voix haute d'Edge, gratuites, sans clé ni installation : Henri, Denise, Rémy, Vivienne, Éloise (fr-FR) et les voix fr-CA, fr-CH, fr-BE, plus de 300 voix dans 74 langues. Le rendu le plus naturel disponible ; nécessite une connexion. * **Voix JARVIS** : deux préréglages en un clic (Paramètres → Voix) : en ligne (Edge Henri) ou hors ligne (Supertonic 3, voix masculine grave, téléchargé automatiquement), timbre « JARVIS » (légèrement plus grave et posé, chaleur, présence, courte réverbération d'intercom), débit calme. diff --git a/package-lock.json b/package-lock.json index d0b0499..ac1ad99 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "eveflow", - "version": "2.5.3", + "version": "2.5.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "eveflow", - "version": "2.5.3", + "version": "2.5.4", "license": "MIT", "dependencies": { "@fontsource/orbitron": "^5.3.0", diff --git a/package.json b/package.json index 28e68d3..d5fed0f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "eveflow", - "version": "2.5.3", - "releaseVersion": "2.5.3", + "version": "2.5.4", + "releaseVersion": "2.5.4", "description": "JARVIS-style desktop HUD for Hermes Agent: voice, streaming runs, scheduled jobs, skills and telemetry", "main": "dist-electron/main.js", "private": true, diff --git a/src/components/settings/SettingsDrawer.tsx b/src/components/settings/SettingsDrawer.tsx index 1e2a994..a2d769d 100644 --- a/src/components/settings/SettingsDrawer.tsx +++ b/src/components/settings/SettingsDrawer.tsx @@ -557,6 +557,13 @@ export function SettingsDrawer({ onClose }: Props) { update({ speech: { autoSpeak: v } })} label="Lire les réponses en streaming" hint="Chaque phrase est prononcée dès qu’elle est complète." /> + update({ speech: { summarizeReplies: v } })} label="Résumé vocal des réponses longues" hint="Seules les premières phrases et la question finale sont lues ; la réponse complète reste dans le fil." /> + {settings.speech.summarizeReplies && ( +
+ + update({ speech: { replySentences: Number(e.target.value) } })} /> +
+ )} update({ speech: { speakIncoming: v } })} label="Lire les messages entrants (webhook, crons)" />
diff --git a/src/lib/text.ts b/src/lib/text.ts index c3956c5..48a80d0 100644 --- a/src/lib/text.ts +++ b/src/lib/text.ts @@ -129,6 +129,33 @@ export function chunkForSpeech(text: string, maxLength = 220): string[] { return out; } +/** Last speakable chunk of a reply when it is a question (« Tu veux que je… ? »), else null. */ +export function closingQuestion(text: string): string | null { + const parts = speakableChunks(text); + const last = parts[parts.length - 1]; + return last && /\?\s*$/.test(last) ? last : null; +} + +/** Chunks of the spoken form of a text (markdown and code already stripped), letters only. */ +function speakableChunks(text: string): string[] { + return chunkForSpeech(cleanForSpeech(text)).filter((part) => /[\p{L}\p{N}]/u.test(part)); +} + +/** + * Spoken digest of a long reply: the first `maxSentences` speakable chunks plus the closing + * question when there is one, so a long answer stays short to listen to but the conversation + * can go on. `truncated` tells the caller that the screen holds more than what is spoken. + */ +export function spokenDigest(text: string, maxSentences: number): { text: string; truncated: boolean } { + const parts = speakableChunks(text); + const limit = Math.max(1, Math.floor(maxSentences)); + if (parts.length <= limit) return { text: parts.join(' '), truncated: false }; + const head = parts.slice(0, limit); + const question = closingQuestion(text); + if (question && !head.includes(question)) head.push(question); + return { text: head.join(' '), truncated: true }; +} + export function previewText(value: string, max = 96): string { const flat = value.replace(/\s+/g, ' ').trim(); return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat; diff --git a/src/services/conversation.ts b/src/services/conversation.ts index 51c302c..963456f 100644 --- a/src/services/conversation.ts +++ b/src/services/conversation.ts @@ -269,8 +269,8 @@ export async function sendMessage(text: string, images: string[] = [], source = if (handle.aborted) { speech.discardStream(); } else if (settings.speech.autoSpeak) { - if (!streamedText && finalText) speech.say(finalText); - else speech.endStream(); + if (!streamedText && finalText) speech.sayReply(finalText); + else speech.endStream(finalText); } else { speech.discardStream(); } diff --git a/src/services/hermes/client.ts b/src/services/hermes/client.ts index 6e2b690..55b9987 100644 --- a/src/services/hermes/client.ts +++ b/src/services/hermes/client.ts @@ -447,6 +447,7 @@ export class HermesClient { let finalText = ''; let streamedText = ''; let completed = false; + let sessionSeen = false; let failure: string | null = null; const handle = await this.streamRunEvents(runId, (event) => { @@ -455,6 +456,7 @@ export class HermesClient { completed = true; finalText = event.text ?? ''; } + if (event.kind === 'session') sessionSeen = true; if (event.kind === 'error') failure = event.message; onEvent(event); }); @@ -465,6 +467,9 @@ export class HermesClient { }); await handle.done; + // The run events never carry the session id; only the run status does. Without adopting it, + // every message would start a fresh Hermes session and the conversation would lose its context. + if (!sessionSeen) await this.adoptRunSession(runId, onEvent); if (stopped || isAborted()) return streamedText; if (failure) throw new Error(failure); @@ -485,6 +490,16 @@ export class HermesClient { return finalText || streamedText; } + /** Reads the session Hermes actually attached to a run and reports it, so the next run continues it. */ + private async adoptRunSession(runId: string, onEvent: SendOptions['onEvent']): Promise { + try { + const info = await this.getRun(runId); + if (info.session_id) onEvent({ kind: 'session', sessionId: String(info.session_id) }); + } catch (err) { + Log.warn('hermes', `run ${runId}: session id unavailable (${(err as Error).message})`); + } + } + private async sendViaSessions(options: SendOptions, setAbort: (fn: () => void) => void, isAborted: () => boolean): Promise { const { onEvent } = options; let sessionId = options.sessionId; diff --git a/src/services/voice/speech.ts b/src/services/voice/speech.ts index e4588bf..aa4cd86 100644 --- a/src/services/voice/speech.ts +++ b/src/services/voice/speech.ts @@ -1,16 +1,23 @@ /** * Singleton facade over the TTS engine bound to the settings store, exposing speaking state - * to the voice and chat stores. + * to the voice and chat stores. Long replies are spoken as a digest (first sentences plus the + * closing question) so listening stays short while the full text remains on screen. */ import { useChat } from '../../state/chat'; import { useSettings } from '../../state/settings'; import { useVoice } from '../../state/voice'; +import { chunkForSpeech, closingQuestion, extractSentences, spokenDigest } from '../../lib/text'; import { TtsEngine } from './tts'; +export const DIGEST_NOTICE = 'Le détail complet est affiché à l’écran.'; + class SpeechFacade { private engine: TtsEngine | null = null; private streaming = false; private streamEnabled = true; + private streamBuffer = ''; + private spokenCount = 0; + private truncated = false; private get tts(): TtsEngine { if (!this.engine) { @@ -35,32 +42,74 @@ class SpeechFacade { return this.engine?.isActive ?? false; } + /** Number of spoken chunks allowed for one reply; unbounded when the digest is off. */ + private replyLimit(): number { + const { summarizeReplies, replySentences } = useSettings.getState().settings.speech; + return summarizeReplies ? Math.max(1, Math.floor(replySentences || 1)) : Number.POSITIVE_INFINITY; + } + say(text: string, options: { interrupt?: boolean } = {}): void { if (useSettings.getState().settings.speech.provider === 'off') return; // While an answer streams, spoken notices are inserted without discarding the rest. this.tts.speak(text, { interrupt: options.interrupt ?? !this.streaming }); } + /** Speak a finished (non-streamed) assistant reply, digested when it is long. */ + sayReply(text: string): void { + const limit = this.replyLimit(); + if (!Number.isFinite(limit)) return this.say(text); + const digest = spokenDigest(text, limit); + this.say(digest.truncated ? `${digest.text} ${DIGEST_NOTICE}` : digest.text); + } + pushStream(delta: string): void { if (!useSettings.getState().settings.speech.autoSpeak) return; this.streaming = true; - if (this.streamEnabled) this.tts.pushStream(delta); + if (!this.streamEnabled) return; + this.streamBuffer += delta; + const { sentences, rest } = extractSentences(this.streamBuffer); + this.streamBuffer = rest; + for (const sentence of sentences) this.enqueueWithinLimit(sentence); } - endStream(): void { - if (this.streaming) this.tts.endStream(); - this.streaming = false; + private enqueueWithinLimit(text: string): void { + if (this.truncated) return; + if (this.spokenCount >= this.replyLimit()) { + this.truncated = true; + return; + } + if (this.tts.enqueue(text)) this.spokenCount++; + } + + /** Flush a streamed reply; `fullText` lets a digested reply end with its closing question. */ + endStream(fullText = ''): void { + if (!this.streaming) return; + const rest = this.streamBuffer.trim(); + if (rest) for (const chunk of chunkForSpeech(rest)) this.enqueueWithinLimit(chunk); + if (this.truncated) { + const question = closingQuestion(fullText); + if (question) this.tts.enqueue(question); + this.tts.enqueue(DIGEST_NOTICE); + } + this.resetStream(); } discardStream(): void { - this.streaming = false; + this.resetStream(); this.tts.stop(); } stop(): void { - this.streaming = false; + this.resetStream(); this.engine?.stop(); } + + private resetStream(): void { + this.streaming = false; + this.streamBuffer = ''; + this.spokenCount = 0; + this.truncated = false; + } } export const speech = new SpeechFacade(); diff --git a/src/services/voice/tts.ts b/src/services/voice/tts.ts index ca3a97c..cceaf83 100644 --- a/src/services/voice/tts.ts +++ b/src/services/voice/tts.ts @@ -115,12 +115,14 @@ export class TtsEngine { if (rest) for (const chunk of chunkForSpeech(rest)) this.enqueue(chunk); } - enqueue(text: string): void { + /** Queue one chunk; false when nothing speakable remains once markdown and code are stripped. */ + enqueue(text: string): boolean { const item = this.makeItem(text); - if (!item) return; + if (!item) return false; this.queue.push(item); this.fillPrefetch(); void this.drain(); + return true; } private makeItem(text: string): QueueItem | null { diff --git a/src/state/settings.ts b/src/state/settings.ts index 8b2d342..9fabace 100644 --- a/src/state/settings.ts +++ b/src/state/settings.ts @@ -33,6 +33,9 @@ export interface VoiceSettings extends SttConfig { export interface SpeechSettings extends TtsConfig { autoSpeak: boolean; speakIncoming: boolean; + /** Speak only the first sentences (and the closing question) of long replies; the full text stays on screen. */ + summarizeReplies: boolean; + replySentences: number; } export interface WebhookSettings { @@ -89,7 +92,7 @@ export const DEFAULT_SETTINGS: Settings = { transport: 'auto', reasoningEffort: '', instructions: - "Réponds en français, de façon concise et orale quand la question est simple; utilise le Markdown uniquement pour le contenu structuré (code, listes, tableaux). Les images doivent être des URL http(s) ou des fichiers du dossier partagé.", + "Réponds en français, de façon concise et orale quand la question est simple; si la réponse est longue, commence par une ou deux phrases qui en donnent l'essentiel, puis le détail. Tu es dans une conversation continue : tiens compte des échanges précédents sans redemander ce qui a déjà été dit. Utilise le Markdown uniquement pour le contenu structuré (code, listes, tableaux). Les images doivent être des URL http(s) ou des fichiers du dossier partagé.", localTools: true, missionModel: '' }, @@ -133,7 +136,9 @@ export const DEFAULT_SETTINGS: Settings = { voiceGender: 'male', timbre: 'jarvis', autoSpeak: true, - speakIncoming: true + speakIncoming: true, + summarizeReplies: true, + replySentences: 4 }, webhook: { enabled: true, port: 7842, secret: '' }, notifications: { quietEnabled: false, quietStart: '22:30', quietEnd: '07:30', priorityKeywords: 'urgent, alerte, alarme, panne', summarizeIncoming: true, summarySentences: 2, nightTheme: true }, diff --git a/tests/hermesSession.test.ts b/tests/hermesSession.test.ts new file mode 100644 index 0000000..283201c --- /dev/null +++ b/tests/hermesSession.test.ts @@ -0,0 +1,57 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { HermesClient } from '../src/services/hermes/client'; +import { httpFetch, httpStream } from '../src/lib/transport'; +import { DEFAULT_SETTINGS } from '../src/state/settings'; +import type { HermesStreamEvent } from '../src/services/hermes/types'; + +vi.mock('../src/lib/transport', async (original) => ({ ...await original(), httpFetch: vi.fn(), httpStream: vi.fn() })); +const config = { ...DEFAULT_SETTINGS.hermes, url: 'https://example.test' }; +afterEach(() => vi.restoreAllMocks()); + +/** Run started, streamed to completion over SSE (which never carries the session id), status read afterwards. */ +function mockRun(sessionId: string) { + const bodies: Array<{ url: string; body: unknown }> = []; + vi.mocked(httpFetch).mockImplementation(async (req) => { + bodies.push({ url: req.url, body: req.body ? JSON.parse(req.body as string) : undefined }); + if (req.url.endsWith('/v1/runs')) return { ok: true, status: 200, statusText: '', headers: {}, text: JSON.stringify({ run_id: 'run_1', status: 'started' }) }; + return { ok: true, status: 200, statusText: '', headers: {}, text: JSON.stringify({ run_id: 'run_1', status: 'completed', session_id: sessionId, output: 'Salut.' }) }; + }); + vi.mocked(httpStream).mockImplementation(async (_req, handlers) => { + // Chunks arrive on a later tick, after the client has accepted the stream (as over IPC). + const done = new Promise((resolve) => setTimeout(() => { + handlers.onChunk('event: message.delta\ndata: {"run_id":"run_1","delta":"Salut."}\n\n'); + handlers.onChunk('event: run.completed\ndata: {"run_id":"run_1","status":"completed","output":"Salut."}\n\n'); + resolve(); + }, 0)); + return { start: { id: 'stream-1', ok: true, status: 200, statusText: '', headers: {} }, done, abort: () => undefined }; + }); + return bodies; +} + +describe('Hermes runs session continuity', () => { + it('adopts the session Hermes attached to the run so the next message continues it', async () => { + const bodies = mockRun('api_abc'); + const events: HermesStreamEvent[] = []; + const client = new HermesClient(config); + const reply = await client.send({ text: 'Bonjour', sessionId: 'eveflow-local', history: [], onEvent: (e) => events.push(e) }, 'runs').result; + expect(reply).toBe('Salut.'); + expect(events.map((e) => e.kind)).toEqual(['run.started', 'delta', 'completed', 'session']); + expect(events[3]).toEqual({ kind: 'session', sessionId: 'api_abc' }); + expect(bodies.map((b) => b.url)).toEqual(['https://example.test/v1/runs', 'https://example.test/v1/runs/run_1']); + expect(bodies[0].body).toEqual(expect.objectContaining({ input: 'Bonjour', session_id: 'eveflow-local' })); + const again = await client.send({ text: 'Et ensuite ?', sessionId: 'api_abc', history: [], onEvent: () => undefined }, 'runs').result; + expect(again).toBe('Salut.'); + expect(bodies[2].body).toEqual(expect.objectContaining({ session_id: 'api_abc' })); + }); + it('does not fail the reply when the run status is unavailable', async () => { + mockRun('api_abc'); + vi.mocked(httpFetch).mockImplementation(async (req) => + req.url.endsWith('/v1/runs') + ? { ok: true, status: 200, statusText: '', headers: {}, text: JSON.stringify({ run_id: 'run_1', status: 'started' }) } + : { ok: false, status: 500, statusText: '', headers: {}, text: 'boom' }); + const events: HermesStreamEvent[] = []; + const reply = await new HermesClient(config).send({ text: 'Bonjour', sessionId: 'x', history: [], onEvent: (e) => events.push(e) }, 'runs').result; + expect(reply).toBe('Salut.'); + expect(events.some((e) => e.kind === 'session')).toBe(false); + }); +}); diff --git a/tests/speechDigest.test.ts b/tests/speechDigest.test.ts new file mode 100644 index 0000000..145ccce --- /dev/null +++ b/tests/speechDigest.test.ts @@ -0,0 +1,59 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { DEFAULT_SETTINGS, useSettings } from '../src/state/settings'; + +const enqueue = vi.fn((text: string) => /[\p{L}\p{N}]/u.test(text.replace(/```[\s\S]*?```/g, ''))); +const speak = vi.fn(); +const stop = vi.fn(); +vi.mock('../src/services/voice/tts', () => ({ + TtsEngine: class { + enqueue = enqueue; + speak = speak; + stop = stop; + onState() { return () => undefined; } + updateConfig() { /* noop */ } + get isActive() { return false; } + } +})); + +const { speech, DIGEST_NOTICE } = await import('../src/services/voice/speech'); + +function stream(text: string, size = 7): void { + for (let i = 0; i < text.length; i += size) speech.pushStream(text.slice(i, i + size)); +} +const spoken = () => enqueue.mock.calls.map((c) => c[0]); + +describe('spoken digest of streamed replies', () => { + beforeEach(() => { + enqueue.mockClear(); + speak.mockClear(); + useSettings.setState({ settings: { ...DEFAULT_SETTINGS, speech: { ...DEFAULT_SETTINGS.speech, summarizeReplies: true, replySentences: 2 } } }); + }); + afterEach(() => speech.stop()); + + it('stops after the configured sentences, then adds the closing question and a notice', () => { + const text = 'Première phrase du rapport. Deuxième phrase utile. Troisième phrase de détail. Quatrième phrase encore. Tu veux la suite ?'; + stream(text); + speech.endStream(text); + expect(spoken()).toEqual(['Première phrase du rapport.', 'Deuxième phrase utile.', 'Tu veux la suite ?', DIGEST_NOTICE]); + }); + + it('speaks short replies in full without any notice', () => { + const text = 'Une phrase courte. Une seconde phrase'; + stream(text); + speech.endStream(text); + expect(spoken()).toEqual(['Une phrase courte.', 'Une seconde phrase']); + }); + + it('speaks everything when the digest is disabled', () => { + useSettings.setState({ settings: { ...DEFAULT_SETTINGS, speech: { ...DEFAULT_SETTINGS.speech, summarizeReplies: false } } }); + const text = 'Première phrase du rapport. Deuxième phrase utile. Troisième phrase de détail. Quatrième phrase encore.'; + stream(text); + speech.endStream(text); + expect(spoken()).toHaveLength(4); + }); + + it('digests a reply delivered in one piece', () => { + speech.sayReply('Première phrase du rapport. Deuxième phrase utile. Troisième phrase de détail. On continue ?'); + expect(speak).toHaveBeenCalledWith(`Première phrase du rapport. Deuxième phrase utile. On continue ? ${DIGEST_NOTICE}`, expect.anything()); + }); +}); diff --git a/tests/text.test.ts b/tests/text.test.ts index 984f18a..07eac9d 100644 --- a/tests/text.test.ts +++ b/tests/text.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { chunkForSpeech, cleanForSpeech, extractSentences, isTranscriptNoise, preprocessMedia } from '../src/lib/text'; +import { chunkForSpeech, cleanForSpeech, closingQuestion, extractSentences, isTranscriptNoise, preprocessMedia, spokenDigest } from '../src/lib/text'; describe('text', () => { it('cleans markdown for speech', () => { @@ -40,3 +40,21 @@ describe('isTranscriptNoise', () => { expect(isTranscriptNoise('Oui')).toBe(false); }); }); + +describe('spokenDigest', () => { + const reply = 'Voici la réponse. Elle contient plusieurs phrases. Une troisième pour la forme. Et une quatrième.\n\n```js\nconsole.log(1)\n```\n\nTu veux que je continue ?'; + it('speaks the first sentences and keeps the closing question', () => { + const digest = spokenDigest(reply, 2); + expect(digest.truncated).toBe(true); + expect(digest.text).toBe('Voici la réponse. Elle contient plusieurs phrases. Tu veux que je continue ?'); + }); + it('leaves short replies untouched', () => { + const digest = spokenDigest('Bonjour Michael. Tout va bien.', 4); + expect(digest).toEqual({ text: 'Bonjour Michael. Tout va bien.', truncated: false }); + }); + it('ignores code blocks when counting and only reports a real question', () => { + expect(spokenDigest('Une phrase.\n\n```\ncode\n```\n', 1).truncated).toBe(false); + expect(closingQuestion(reply)).toBe('Tu veux que je continue ?'); + expect(closingQuestion('Aucune question ici.')).toBeNull(); + }); +});