feat: add adaptive TTS text-to-speech synchronization for Reels with real-time sync info widget

Calculate optimal word_duration based on actual TTS audio duration and text punctuation.

- server/services/ttsSync.ts: new service to measure TTS audio and compute sync timing

- server/routes/reels.ts: integrate sync into preview and background processing

- client: auto-calculate sync info widget in desktop and mobile Reel creation

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

client/src/pages/mobile/new-reel.tsx

@@ -1,4 +1,4 @@
-import { useState, useCallback, useRef } from "react";
+import { useState, useCallback, useRef, useEffect } from "react";
 import { useQuery, useMutation } from "@tanstack/react-query";
 import { useLocation } from "wouter";
 import {
@@ -65,6 +65,33 @@ export default function MobileNewReel() {
     const [stabilize, setStabilize] = useState(true); // Activé par défaut pour les Reels
     const [enableEndingEffect, setEnableEndingEffect] = useState(true);
 
+    // TTS Sync state
+    const [syncInfo, setSyncInfo] = useState<{
+        wordDuration: number;
+        audioDuration: number;
+        wordCount: number;
+        isHealthy: boolean;
+        warnings: string[];
+    } | null>(null);
+
+    // Auto-calculate TTS sync
+    useEffect(() => {
+        if (!ttsEnabled || !overlayText.trim()) {
+            setSyncInfo(null);
+            return;
+        }
+        const timer = setTimeout(() => {
+            apiRequest('POST', '/api/reels/sync-info', {
+                text: overlayText,
+                voice: ttsVoice,
+            })
+                .then(r => r.json())
+                .then(data => setSyncInfo(data))
+                .catch(() => setSyncInfo(null));
+        }, 800);
+        return () => clearTimeout(timer);
+    }, [overlayText, ttsVoice, ttsEnabled]);
+
 
     const { data: pages = [] } = useQuery<SocialPage[]>({
         queryKey: ['/api/pages'],
@@ -475,6 +502,23 @@ export default function MobileNewReel() {
                                             <Play className="w-3 h-3 mr-1" /> Tester la voix
                                         </Button>
                                     </div>
+
+                                    {syncInfo && (
+                                        <div className={`p-2 rounded-lg border mt-2 ${syncInfo.isHealthy ? 'bg-green-500/10 border-green-500/30' : 'bg-yellow-500/10 border-yellow-500/30'}`}>
+                                            <div className="flex items-center justify-between text-xs">
+                                                <span className="font-medium">Sync</span>
+                                                <span className={syncInfo.isHealthy ? 'text-green-600' : 'text-yellow-600'}>
+                                                    {syncInfo.wordCount} mots · {syncInfo.audioDuration.toFixed(1)}s
+                                                </span>
+                                            </div>
+                                            {syncInfo.warnings.map((w, i) => (
+                                                <p key={i} className="text-[10px] text-yellow-600 mt-0.5">⚠️ {w}</p>
+                                            ))}
+                                            {syncInfo.isHealthy && syncInfo.warnings.length === 0 && (
+                                                <p className="text-[10px] text-green-600 mt-0.5">✅ Timing OK</p>
+                                            )}
+                                        </div>
+                                    )}
                                 </div>
                             )}
                         </div>

client/src/pages/new-reel.tsx

@@ -70,6 +70,15 @@ export default function NewReel() {
     const [stabilize, setStabilize] = useState(true); // default to true
     const [enableEndingEffect, setEnableEndingEffect] = useState(true);
 
+    // TTS Sync state
+    const [syncInfo, setSyncInfo] = useState<{
+        wordDuration: number;
+        audioDuration: number;
+        wordCount: number;
+        isHealthy: boolean;
+        warnings: string[];
+    } | null>(null);
+
     // Enable TTS by default on mobile
     useEffect(() => {
         const isMobile = window.innerWidth < 768;
@@ -78,6 +87,24 @@ export default function NewReel() {
         }
     }, []);
 
+    // Auto-calculate TTS sync when text or voice changes
+    useEffect(() => {
+        if (!ttsEnabled || !overlayText.trim()) {
+            setSyncInfo(null);
+            return;
+        }
+        const timer = setTimeout(() => {
+            apiRequest('POST', '/api/reels/sync-info', {
+                text: overlayText,
+                voice: ttsVoice,
+            })
+                .then(r => r.json())
+                .then(data => setSyncInfo(data))
+                .catch(() => setSyncInfo(null));
+        }, 800);
+        return () => clearTimeout(timer);
+    }, [overlayText, ttsVoice, ttsEnabled]);
+
 
     // État audio preview
     const [isPlaying, setIsPlaying] = useState<string | null>(null);
@@ -796,6 +823,23 @@ export default function NewReel() {
                                                         </Button>
                                                     </div>
 
+                                                    {syncInfo && (
+                                                        <div className={`p-3 rounded-lg border mt-3 ${syncInfo.isHealthy ? 'bg-green-500/10 border-green-500/30' : 'bg-yellow-500/10 border-yellow-500/30'}`}>
+                                                            <div className="flex items-center justify-between text-sm">
+                                                                <span className="font-medium">Sync texte/voix</span>
+                                                                <span className={syncInfo.isHealthy ? 'text-green-600' : 'text-yellow-600'}>
+                                                                    {syncInfo.wordCount} mots · {syncInfo.audioDuration.toFixed(1)}s · {syncInfo.wordDuration.toFixed(2)}s/mot
+                                                                </span>
+                                                            </div>
+                                                            {syncInfo.warnings.map((w, i) => (
+                                                                <p key={i} className="text-xs text-yellow-600 mt-1">⚠️ {w}</p>
+                                                            ))}
+                                                            {syncInfo.isHealthy && syncInfo.warnings.length === 0 && (
+                                                                <p className="text-xs text-green-600 mt-1">✅ Timing calculé automatiquement</p>
+                                                            )}
+                                                        </div>
+                                                    )}
+
                                                     <p className="text-xs text-muted-foreground mt-2">
                                                         Le texte sera automatiquement synchronisé avec la voix.
                                                         Les #hashtags et émojis ne seront pas lus.

server/routes/reels.ts

@@ -14,6 +14,7 @@ import { db } from '../db';
 import { cloudinaryConfig } from '@shared/schema';
 import { eq } from 'drizzle-orm';
 
+import { ttsSyncService } from '../services/ttsSync';
 export const reelsRouter = Router();
 
 // ============================================
@@ -349,13 +350,25 @@ reelsRouter.post('/reels/preview', async (req: Request, res: Response) => {
             console.error('Error fetching watermark configuration', e);
         }
 
+        // Calculer le word_duration synchronisé si TTS activé
+        let finalWordDuration = wordDuration;
+        if (ttsEnabled && overlayText) {
+            try {
+                const sync = await ttsSyncService.calculateSyncTiming(overlayText, ttsVoice);
+                finalWordDuration = sync.wordDuration;
+                console.log(`🎯 TTS Sync (preview): ${sync.wordCount} mots, ${sync.audioDuration.toFixed(2)}s audio, word_duration=${finalWordDuration.toFixed(3)}s`);
+            } catch (syncErr) {
+                console.warn('⚠️ TTS sync failed (preview), using default word_duration:', syncErr);
+            }
+        }
+
         // Traiter la vidéo via FFmpeg
         const result = await ffmpegService.processReelFromUrl(resolveInternalUrl(media.originalUrl), {
             text: overlayText,
             musicUrl: finalMusicUrl,
             ttsEnabled,
             ttsVoice,
-            wordDuration,
+            wordDuration: finalWordDuration,
             fontSize,
             musicVolume,
             drawText,
@@ -405,6 +418,24 @@ reelsRouter.post('/reels/tts-preview', async (req: Request, res: Response) => {
     }
 });
 
+/**
+ * Calculer la synchronisation texte/voix TTS
+ * POST /api/reels/sync-info
+ */
+reelsRouter.post('/reels/sync-info', async (req: Request, res: Response) => {
+    try {
+        const { text, voice } = req.body;
+        if (!text || !voice) {
+            return res.status(400).json({ error: 'Texte et voix requis' });
+        }
+        const sync = await ttsSyncService.calculateSyncTiming(text, voice);
+        res.json(sync);
+    } catch (error) {
+        console.error('❌ Error calculating sync:', error);
+        res.status(500).json({ error: 'Erreur de calcul de synchronisation' });
+    }
+});
+
 /**
  * Traitement d'arrière-plan pour les Reels
  * Gère le pipeline FFmpeg -> Cloudinary -> Facebook de manière asynchrone
@@ -562,12 +593,24 @@ async function processReelBackground(
             console.error('Error fetching watermark configuration', e);
         }
 
+        // Calculer le word_duration synchronisé si TTS activé
+        let finalWordDuration = wordDuration ?? 0.6;
+        if (ttsEnabled && overlayText && ttsVoice) {
+            try {
+                const sync = await ttsSyncService.calculateSyncTiming(overlayText, ttsVoice);
+                finalWordDuration = sync.wordDuration;
+                console.log(`🎯 TTS Sync (background): ${sync.wordCount} mots, ${sync.audioDuration.toFixed(2)}s audio, word_duration=${finalWordDuration.toFixed(3)}s`);
+            } catch (syncErr) {
+                console.warn('⚠️ TTS sync failed (background), using default word_duration:', syncErr);
+            }
+        }
+
         const ffmpegResult = await ffmpegService.processReelFromUrl(resolveInternalUrl(media.originalUrl), {
             text: overlayText,
             musicUrl: finalMusicUrl,
             ttsEnabled,
             ttsVoice,
-            wordDuration,
+            wordDuration: finalWordDuration,
             fontSize,
             musicVolume,
             drawText,

server/services/ttsSync.ts

@@ -0,0 +1,89 @@
+import { ffmpegService } from './ffmpeg';
+import * as musicMetadata from 'music-metadata';
+
+export interface SyncTiming {
+  wordDuration: number;
+  audioDuration: number;
+  wordCount: number;
+  punctuationPause: number;
+  isHealthy: boolean;
+  warnings: string[];
+}
+
+export class TtsSyncService {
+  /**
+   * Calcule le word_duration optimal pour synchroniser l'affichage du texte
+   * avec la durée réelle de la voix TTS générée.
+   */
+  async calculateSyncTiming(text: string, voice: string): Promise<SyncTiming> {
+    const cleanTtsText = this.cleanForTts(text);
+    const cleanDisplayText = this.cleanForDisplay(text);
+
+    // 1. Générer le TTS preview et mesurer sa durée exacte
+    const ttsResult = await ffmpegService.previewTTS(cleanTtsText, voice);
+    if (!ttsResult.success || !ttsResult.audioBase64) {
+      throw new Error('TTS preview failed: ' + (ttsResult.error || 'unknown'));
+    }
+
+    const audioBuffer = Buffer.from(ttsResult.audioBase64, 'base64');
+    const metadata = await musicMetadata.parseBuffer(audioBuffer, 'audio/mpeg');
+    const audioDuration = metadata.format.duration || 0;
+
+    // 2. Analyser le texte
+    const words = cleanDisplayText.split(/\s+/).filter(w => w.length > 0);
+    const wordCount = words.length;
+    const punctuationPause = this.calculatePunctuationPauses(cleanDisplayText);
+
+    // 3. Calculer le word_duration ajusté
+    const effectiveDuration = Math.max(audioDuration - punctuationPause, 0.5);
+    const wordDuration = wordCount > 0 ? effectiveDuration / wordCount : 0.6;
+
+    // 4. Validation
+    const warnings: string[] = [];
+    const isHealthy = wordDuration >= 0.25 && wordDuration <= 1.5;
+    if (wordDuration < 0.25) {
+      warnings.push('Texte trop long : les mots défileront très vite. Envisagez de raccourcir.');
+    }
+    if (wordDuration > 1.2) {
+      warnings.push('Texte très court : affichage lent.');
+    }
+
+    return {
+      wordDuration,
+      audioDuration,
+      wordCount,
+      punctuationPause,
+      isHealthy,
+      warnings,
+    };
+  }
+
+  private cleanForTts(text: string): string {
+    return text
+      .replace(/#\w+/g, '')
+      .replace(/https?:\/\/\S+/g, '')
+      .replace(/[\uD83C-\uD83E][\uDC00-\uDFFF]|[☀-⛿✀-➿]/g, '')
+      .trim();
+  }
+
+  private cleanForDisplay(text: string): string {
+    return text.trim();
+  }
+
+  private calculatePunctuationPauses(text: string): number {
+    let pause = 0;
+    const chars = text.split('');
+    for (const c of chars) {
+      if (',;'.includes(c)) pause += 0.3;
+      else if ('.!?'.includes(c)) pause += 0.6;
+      else if (':'.includes(c)) pause += 0.4;
+      else if (c === '\n') pause += 0.3;
+    }
+    // "..." compte comme un seul point mais pause plus longue
+    const ellipsisCount = (text.match(/\.\.\./g) || []).length;
+    pause += ellipsisCount * 0.3; // bonus pour les points de suspension
+    return pause;
+  }
+}
+
+export const ttsSyncService = new TtsSyncService();