From 24a1d39e886149512b85841b74e904b5ba7cd65e Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Tue, 24 Mar 2026 16:44:06 +0100 Subject: [PATCH] feat: Implement Remotion video generation feature, allowing users to combine images with text overlay and text-to-speech. --- client/src/pages/mobile/remotion-video.tsx | 426 +++++++++++++-------- client/src/pages/remotion-video.tsx | 190 +++++---- client/src/remotion/ImageComposition.tsx | 101 +++-- server/routes/remotion.ts | 15 +- 4 files changed, 479 insertions(+), 253 deletions(-) diff --git a/client/src/pages/mobile/remotion-video.tsx b/client/src/pages/mobile/remotion-video.tsx index 861b734..b666f85 100644 --- a/client/src/pages/mobile/remotion-video.tsx +++ b/client/src/pages/mobile/remotion-video.tsx @@ -1,48 +1,82 @@ -import { useState, useMemo } from "react"; +import { useState, useMemo, useRef } from "react"; import { Player } from "@remotion/player"; import { ImageComposition } from "@/remotion/ImageComposition"; import { useToast } from "@/hooks/use-toast"; -import { Card, CardContent, CardHeader, CardTitle, CardDescription } from "@/components/ui/card"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Button } from "@/components/ui/button"; -import { UploadCloud, Video, Loader2, Menu, Check } from "lucide-react"; +import { Textarea } from "@/components/ui/textarea"; +import { Label } from "@/components/ui/label"; +import { Switch } from "@/components/ui/switch"; +import { Slider } from "@/components/ui/slider"; +import { UploadCloud, Video, Loader2, Menu, Check, Sparkles, Mic, Volume2, Music, Play, Pause } from "lucide-react"; import { Sheet, SheetContent, SheetTrigger } from "@/components/ui/sheet"; import Sidebar from "@/components/sidebar"; -import { useQuery } from "@tanstack/react-query"; +import { useQuery, useMutation } from "@tanstack/react-query"; +import { apiRequest } from "@/lib/queryClient"; import type { Media } from "@shared/schema"; +interface AudioTrack { id: string; title: string; fileName: string; url: string; duration: number; } + +const TTS_VOICES = [ + { id: "fr-FR-VivienneMultilingualNeural", label: "Vivienne", type: "Femme" }, + { id: "fr-FR-RemyMultilingualNeural", label: "Rémy", type: "Homme" }, + { id: "fr-FR-DeniseNeural", label: "Denise", type: "Femme" }, + { id: "fr-FR-HenriNeural", label: "Henri", type: "Homme" }, + { id: "fr-FR-EloiseNeural", label: "Éloïse", type: "Enfant" }, +]; + +function stripForTTS(text: string) { + return text.replace(/#\w+/g, '').replace(/[\uD800-\uDFFF]/g, '').replace(/[\u2600-\u27BF]/g, '').replace(/\s+/g, ' ').trim(); +} + +function estimateWordTimings(text: string, fps = 30, wps = 2.5) { + const words = text.split(/\s+/).filter(Boolean); + const framesPerWord = fps / wps; + let frame = 0; + return words.map(word => { + const isSpoken = !/^#/.test(word) && !(/[\uD800-\uDFFF\u2600-\u27BF]/.test(word)); + const dur = isSpoken ? framesPerWord : framesPerWord * 0.4; + const start = frame; + frame += Math.round(dur); + return { word, startFrame: start, endFrame: frame }; + }); +} + export default function MobileRemotionVideoPage() { const [images, setImages] = useState([]); const [selectedLibraryImages, setSelectedLibraryImages] = useState([]); + const [overlayText, setOverlayText] = useState(""); + const [productInfo, setProductInfo] = useState(""); + const [generatedVariants, setGeneratedVariants] = useState([]); + const [ttsEnabled, setTtsEnabled] = useState(true); + const [ttsVoice, setTtsVoice] = useState("fr-FR-VivienneMultilingualNeural"); + const [musicFile, setMusicFile] = useState(null); + const [selectedTrack, setSelectedTrack] = useState(null); + const [musicVolume, setMusicVolume] = useState(0.3); + const [isPlaying, setIsPlaying] = useState(null); const [isRendering, setIsRendering] = useState(false); const [videoUrl, setVideoUrl] = useState(null); + const audioRef = useRef(null); const { toast } = useToast(); - const { data: allMedia = [] } = useQuery({ - queryKey: ['/api/media'], - }); + const { data: allMedia = [] } = useQuery({ queryKey: ['/api/media'] }); + const { data: audioTracks = [], isLoading: tracksLoading } = useQuery({ queryKey: ['/api/audio-tracks'] }); const imageMediaList = allMedia.filter(m => m.type === 'image').slice(0, 20); + const generateTextMutation = useMutation({ + mutationFn: async (text: string) => (await apiRequest('POST', '/api/reels/generate-text', { productInfo: text })).json(), + onSuccess: (data: any) => { setGeneratedVariants(data.variants || []); toast({ title: "Textes générés" }); }, + onError: () => toast({ title: "Erreur IA", variant: "destructive" }), + }); + const handleFileChange = (e: React.ChangeEvent) => { - if (e.target.files) { - const files = Array.from(e.target.files); - const remainingSlots = 4 - selectedLibraryImages.length; - if (files.length > remainingSlots) { - toast({ title: "Limite atteinte", description: `Vous ne pouvez ajouter que ${remainingSlots} image(s) supplémentaire(s).`, variant: "destructive" }); - } - setImages(files.slice(0, remainingSlots)); - } + if (e.target.files) setImages(Array.from(e.target.files).slice(0, 4 - selectedLibraryImages.length)); }; const toggleLibraryImage = (media: Media) => { setSelectedLibraryImages(prev => { - const isSelected = prev.find(m => m.id === media.id); - if (isSelected) { - return prev.filter(m => m.id !== media.id); - } - if (images.length + prev.length >= 4) { - toast({ title: "Limite", description: "Vous pouvez sélectionner 4 images maximum au total.", variant: "destructive" }); - return prev; - } + if (prev.find(m => m.id === media.id)) return prev.filter(m => m.id !== media.id); + if (images.length + prev.length >= 4) { toast({ title: "4 images max.", variant: "destructive" }); return prev; } return [...prev, media]; }); }; @@ -55,176 +89,250 @@ export default function MobileRemotionVideoPage() { }, [images, selectedLibraryImages]); const totalSelected = images.length + selectedLibraryImages.length; + const previewWordTimings = useMemo(() => overlayText ? estimateWordTimings(overlayText) : undefined, [overlayText]); + const totalFrames = useMemo(() => { + if (previewWordTimings?.length) + return Math.max(combinedUrls.length * 3 * 30, previewWordTimings[previewWordTimings.length - 1].endFrame + 30); + return combinedUrls.length > 0 ? combinedUrls.length * 3 * 30 : 300; + }, [previewWordTimings, combinedUrls]); + + const handleTtsPreview = async () => { + const ttsText = stripForTTS(overlayText); + if (!ttsText) return; + try { + const r = await apiRequest('POST', '/api/reels/tts-preview', { text: ttsText, voice: ttsVoice }); + const data = await r.json(); + if (data.success && data.audioBase64) new window.Audio(`data:audio/mp3;base64,${data.audioBase64}`).play(); + } catch { toast({ title: "Erreur prévisualisation voix", variant: "destructive" }); } + }; + + const togglePlayPreview = (track: AudioTrack) => { + if (!audioRef.current) return; + if (isPlaying === track.id) { audioRef.current.pause(); setIsPlaying(null); } + else { audioRef.current.src = track.url; audioRef.current.play(); setIsPlaying(track.id); } + }; const handleGenerate = async () => { - if (totalSelected < 3) { - toast({ - title: "Pas assez d'images", - description: "Veuillez sélectionner au moins 3 images.", - variant: "destructive" - }); - return; - } - + if (totalSelected < 3) { toast({ title: "Minimum 3 images", variant: "destructive" }); return; } setIsRendering(true); setVideoUrl(null); - try { const formData = new FormData(); images.forEach(img => formData.append("images", img)); selectedLibraryImages.forEach(m => formData.append("existingImageUrls", m.originalUrl)); - - const response = await fetch("/api/remotion/render", { - method: "POST", - body: formData, - }); - - if (!response.ok) { - throw new Error("Erreur de génération"); - } - + if (overlayText) formData.append("overlayText", overlayText); + if (ttsEnabled && overlayText) formData.append("ttsVoice", ttsVoice); + if (musicFile) { formData.append("music", musicFile); formData.append("musicVolume", String(musicVolume)); } + else if (selectedTrack) { formData.append("musicTrackUrl", selectedTrack.url); formData.append("musicVolume", String(musicVolume)); } + const response = await fetch("/api/remotion/render", { method: "POST", body: formData }); + if (!response.ok) { const e = await response.json().catch(() => ({})); throw new Error(e.error || "Erreur"); } const data = await response.json(); setVideoUrl(data.url); - - toast({ - title: "Vidéo générée !", - description: "Votre vidéo Remotion est prête.", - }); - - } catch (err) { - toast({ - title: "Erreur", - description: "Impossible de créer la vidéo.", - variant: "destructive" - }); - } finally { - setIsRendering(false); - } + toast({ title: "Vidéo générée !" }); + } catch (err: any) { + toast({ title: "Erreur", description: err.message, variant: "destructive" }); + } finally { setIsRendering(false); } }; - // 3s per image, 30fps - const totalFrames = combinedUrls.length > 0 ? combinedUrls.length * 3 * 30 : 300; - return ( -
- {/* Header Mobile */} -
-
- - - - - - - - -

Générateur Remotion

-
+
+