feat: add adaptive TTS text-to-speech synchronization for Reels with real-time sync info widget

Calculate optimal word_duration based on actual TTS audio duration and text punctuation.

- server/services/ttsSync.ts: new service to measure TTS audio and compute sync timing

- server/routes/reels.ts: integrate sync into preview and background processing

- client: auto-calculate sync info widget in desktop and mobile Reel creation

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
MichaelandClaude Opus 4.7 committed 2026-05-04 12:35:04 +02:00
1 parent 9913c307ed
commit 9ce735dc23
5 files changed
+1359 -3

No files matched your search

+45 -2
View File
@@ -14,6 +14,7 @@ import { db } from '../db';
import { cloudinaryConfig } from '@shared/schema';
import { eq } from 'drizzle-orm';
import { ttsSyncService } from '../services/ttsSync';
export const reelsRouter = Router();
// ============================================
@@ -349,13 +350,25 @@ reelsRouter.post('/reels/preview', async (req: Request, res: Response) => {
console.error('Error fetching watermark configuration', e);
}
// Calculer le word_duration synchronisé si TTS activé
let finalWordDuration = wordDuration;
if (ttsEnabled && overlayText) {
try {
const sync = await ttsSyncService.calculateSyncTiming(overlayText, ttsVoice);
finalWordDuration = sync.wordDuration;
console.log(`🎯 TTS Sync (preview): ${sync.wordCount} mots, ${sync.audioDuration.toFixed(2)}s audio, word_duration=${finalWordDuration.toFixed(3)}s`);
} catch (syncErr) {
console.warn('⚠️ TTS sync failed (preview), using default word_duration:', syncErr);
}
}
// Traiter la vidéo via FFmpeg
const result = await ffmpegService.processReelFromUrl(resolveInternalUrl(media.originalUrl), {
text: overlayText,
musicUrl: finalMusicUrl,
ttsEnabled,
ttsVoice,
wordDuration,
wordDuration: finalWordDuration,
fontSize,
musicVolume,
drawText,
@@ -405,6 +418,24 @@ reelsRouter.post('/reels/tts-preview', async (req: Request, res: Response) => {
}
});
/**
* Calculer la synchronisation texte/voix TTS
* POST /api/reels/sync-info
*/
reelsRouter.post('/reels/sync-info', async (req: Request, res: Response) => {
try {
const { text, voice } = req.body;
if (!text || !voice) {
return res.status(400).json({ error: 'Texte et voix requis' });
}
const sync = await ttsSyncService.calculateSyncTiming(text, voice);
res.json(sync);
} catch (error) {
console.error('❌ Error calculating sync:', error);
res.status(500).json({ error: 'Erreur de calcul de synchronisation' });
}
});
/**
* Traitement d'arrière-plan pour les Reels
* Gère le pipeline FFmpeg -> Cloudinary -> Facebook de manière asynchrone
@@ -562,12 +593,24 @@ async function processReelBackground(
console.error('Error fetching watermark configuration', e);
}
// Calculer le word_duration synchronisé si TTS activé
let finalWordDuration = wordDuration ?? 0.6;
if (ttsEnabled && overlayText && ttsVoice) {
try {
const sync = await ttsSyncService.calculateSyncTiming(overlayText, ttsVoice);
finalWordDuration = sync.wordDuration;
console.log(`🎯 TTS Sync (background): ${sync.wordCount} mots, ${sync.audioDuration.toFixed(2)}s audio, word_duration=${finalWordDuration.toFixed(3)}s`);
} catch (syncErr) {
console.warn('⚠️ TTS sync failed (background), using default word_duration:', syncErr);
}
}
const ffmpegResult = await ffmpegService.processReelFromUrl(resolveInternalUrl(media.originalUrl), {
text: overlayText,
musicUrl: finalMusicUrl,
ttsEnabled,
ttsVoice,
wordDuration,
wordDuration: finalWordDuration,
fontSize,
musicVolume,
drawText,