mirror of
https://github.com/R0m1k3/Socialflow.git
synced 2026-10-17 04:09:51 +02:00
feat: Replace external storage with local file storage + auto cleanup
Media is now stored directly on the server disk (no external service). Files are served via Express static middleware at /uploads/*. Cleanup rules: - Videos deleted immediately after successful publish to Facebook/Instagram - Videos older than 7 days purged daily by cron job - Images older than 30 days purged daily by cron job Changes: - server/services/minio.ts: replaced S3 client with local fs read/write - server/index.ts: added static serving for /uploads/media, /logos, /stories - server/services/media-purge.ts: new rules (images 30d, videos 7d) - server/services/scheduler.ts: delete local videos after successful publish - docker-compose.yml: add media_uploads, logos_uploads, stories_uploads volumes - settings UI: removed credentials card, shows local storage info - package.json: removed googleapis (unneeded) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
1 parent
ba94e7cc7d
commit
7733acbc22
9 files changed
+145
-378
No files matched your search
@@ -66,6 +66,17 @@ app.use('/api/auth/login', authLimiter);
|
||||
const AUDIO_UPLOAD_DIR = process.env.AUDIO_UPLOAD_DIR || '/app/uploads/audio';
|
||||
app.use('/uploads/audio', express.static(AUDIO_UPLOAD_DIR));
|
||||
|
||||
// Serve locally stored media files (images, videos, logos, stories)
|
||||
const MEDIA_UPLOAD_DIR = path.join(process.cwd(), 'uploads', 'media');
|
||||
const LOGOS_UPLOAD_DIR = path.join(process.cwd(), 'uploads', 'logos');
|
||||
const STORIES_UPLOAD_DIR = path.join(process.cwd(), 'uploads', 'stories');
|
||||
try { fs.mkdirSync(MEDIA_UPLOAD_DIR, { recursive: true }); } catch { /* ignore */ }
|
||||
try { fs.mkdirSync(LOGOS_UPLOAD_DIR, { recursive: true }); } catch { /* ignore */ }
|
||||
try { fs.mkdirSync(STORIES_UPLOAD_DIR, { recursive: true }); } catch { /* ignore */ }
|
||||
app.use('/uploads/media', express.static(MEDIA_UPLOAD_DIR));
|
||||
app.use('/uploads/logos', express.static(LOGOS_UPLOAD_DIR));
|
||||
app.use('/uploads/stories', express.static(STORIES_UPLOAD_DIR));
|
||||
|
||||
// Serve temp uploads for Remotion
|
||||
const TEMP_UPLOAD_DIR = process.env.TEMP_UPLOAD_DIR || path.join(process.cwd(), 'uploads', 'temp');
|
||||
try { fs.mkdirSync(TEMP_UPLOAD_DIR, { recursive: true }); } catch { /* ignore */ }
|
||||
|
||||
+4
-4
@@ -1679,11 +1679,11 @@ export async function registerRoutes(app: Express): Promise<Server> {
|
||||
return res.status(400).json({ error: "No logo file uploaded" });
|
||||
}
|
||||
|
||||
// Check if Cloudinary is configured globally
|
||||
const cloudinaryConfig = await storage.getCloudinaryConfig();
|
||||
// Get or auto-create config row (local storage doesn't need credentials)
|
||||
let cloudinaryConfig = await storage.getCloudinaryConfig();
|
||||
if (!cloudinaryConfig) {
|
||||
return res.status(400).json({
|
||||
error: "Cloudinary not configured. Please configure Cloudinary first."
|
||||
cloudinaryConfig = await storage.createCloudinaryConfig({
|
||||
cloudName: 'local', apiKey: 'local', apiSecret: 'local',
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -1,64 +1,58 @@
|
||||
import { db } from '../db';
|
||||
import { media } from '@shared/schema';
|
||||
import { eq, lt, isNotNull, and } from 'drizzle-orm';
|
||||
import { minioService as cloudinaryService } from './minio';
|
||||
import { eq, lt, and } from 'drizzle-orm';
|
||||
import { minioService as storageService } from './minio';
|
||||
|
||||
/**
|
||||
* Automates the cleanup of old media to save Cloudinary storage space.
|
||||
* Scans the database for media older than 60 days, deletes the physical
|
||||
* files from Cloudinary, and removes the database records (cascading to post_media).
|
||||
* Purge old local media to save disk space:
|
||||
* - Images older than 30 days
|
||||
* - Videos older than 7 days
|
||||
*/
|
||||
export async function purgeOldMedia() {
|
||||
console.log('[Media Purge] Starting automatic purge of old media...');
|
||||
console.log('[Media Purge] Starting automatic purge of old media...');
|
||||
|
||||
try {
|
||||
const sixtyDaysAgo = new Date();
|
||||
sixtyDaysAgo.setDate(sixtyDaysAgo.getDate() - 60);
|
||||
try {
|
||||
const thirtyDaysAgo = new Date();
|
||||
thirtyDaysAgo.setDate(thirtyDaysAgo.getDate() - 30);
|
||||
|
||||
// Find all media older than 60 days that have a cloudinaryPublicId
|
||||
const oldMedia = await db.query.media.findMany({
|
||||
where: (media, { and, lt, isNotNull }) => and(
|
||||
lt(media.createdAt, sixtyDaysAgo),
|
||||
isNotNull(media.cloudinaryPublicId)
|
||||
)
|
||||
});
|
||||
const sevenDaysAgo = new Date();
|
||||
sevenDaysAgo.setDate(sevenDaysAgo.getDate() - 7);
|
||||
|
||||
if (oldMedia.length === 0) {
|
||||
console.log('[Media Purge] No old media found to purge.');
|
||||
return;
|
||||
}
|
||||
const oldImages = await db.query.media.findMany({
|
||||
where: (m, { and, lt, eq }) => and(eq(m.type, 'image'), lt(m.createdAt, thirtyDaysAgo))
|
||||
});
|
||||
|
||||
console.log(`[Media Purge] Found ${oldMedia.length} media items older than 60 days to purge.`);
|
||||
const oldVideos = await db.query.media.findMany({
|
||||
where: (m, { and, lt, eq }) => and(eq(m.type, 'video'), lt(m.createdAt, sevenDaysAgo))
|
||||
});
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
const toDelete = [...oldImages, ...oldVideos];
|
||||
|
||||
for (const item of oldMedia) {
|
||||
if (!item.cloudinaryPublicId) continue;
|
||||
|
||||
try {
|
||||
// Delete from Cloudinary (using global config, userId is passed but ignored internally)
|
||||
await cloudinaryService.deleteMedia(
|
||||
item.cloudinaryPublicId,
|
||||
item.userId,
|
||||
item.type === 'video' ? 'video' : 'image'
|
||||
);
|
||||
|
||||
// Delete from Database (cascades to post_media, effectively leaving the post text intact)
|
||||
await db.delete(media).where(eq(media.id, item.id));
|
||||
|
||||
successCount++;
|
||||
console.log(`[Media Purge] Successfully deleted media ${item.id} (Cloudinary ID: ${item.cloudinaryPublicId})`);
|
||||
} catch (err) {
|
||||
// Cloudinary file might already be deleted or missing, but we still might want to clean DB if it's consistently failing.
|
||||
// For safety, we only delete DB row if Cloudinary deletion succeeds or throws 'not found' (which we assume is fine).
|
||||
console.error(`[Media Purge] Failed to delete media ${item.id} from Cloudinary:`, err);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[Media Purge] Purge completed. Successfully deleted: ${successCount}, Failed: ${errorCount}.`);
|
||||
} catch (error) {
|
||||
console.error('[Media Purge] Fatal error during media purge process:', error);
|
||||
if (toDelete.length === 0) {
|
||||
console.log('[Media Purge] No old media found to purge.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[Media Purge] Found ${oldImages.length} old images (>30 days) and ${oldVideos.length} old videos (>7 days) to purge.`);
|
||||
|
||||
let successCount = 0;
|
||||
let errorCount = 0;
|
||||
|
||||
for (const item of toDelete) {
|
||||
try {
|
||||
if (item.cloudinaryPublicId) {
|
||||
await storageService.deleteMedia(item.cloudinaryPublicId, item.userId, item.type === 'video' ? 'video' : 'image');
|
||||
}
|
||||
await db.delete(media).where(eq(media.id, item.id));
|
||||
successCount++;
|
||||
} catch (err) {
|
||||
console.error(`[Media Purge] Failed to delete media ${item.id}:`, err);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`[Media Purge] Purge completed. Deleted: ${successCount}, Failed: ${errorCount}.`);
|
||||
} catch (error) {
|
||||
console.error('[Media Purge] Fatal error during media purge process:', error);
|
||||
}
|
||||
}
|
||||
+44
-67
@@ -1,33 +1,33 @@
|
||||
import { S3Client, PutObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';
|
||||
import { storage } from '../storage';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
function getS3Client(accessKey: string, secretKey: string, endpointUrl?: string | null): S3Client {
|
||||
const endpoint = endpointUrl || process.env.MINIO_ENDPOINT || 'http://localhost:9000';
|
||||
return new S3Client({
|
||||
endpoint,
|
||||
region: 'us-east-1',
|
||||
credentials: { accessKeyId: accessKey, secretAccessKey: secretKey },
|
||||
forcePathStyle: true,
|
||||
});
|
||||
const UPLOADS_BASE = path.resolve(process.cwd(), 'uploads');
|
||||
|
||||
function ensureDir(dir: string): void {
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
|
||||
function getAppUrl(): string {
|
||||
return (process.env.APP_URL || 'http://localhost:5555').replace(/\/$/, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the public URL for a stored object.
|
||||
* Uses DB publicUrl > MINIO_PUBLIC_URL env > MINIO_ENDPOINT env > localhost fallback.
|
||||
* Builds the public URL for a locally stored file.
|
||||
* objectKey is the relative path from uploads/ (e.g. "media/123-file.jpg").
|
||||
* bucketName and publicUrlOverride are ignored (local storage compatibility shim).
|
||||
*/
|
||||
export function buildMinioUrl(bucketName: string, objectKey: string, publicUrlOverride?: string | null): string {
|
||||
const base = (publicUrlOverride || process.env.MINIO_PUBLIC_URL || process.env.MINIO_ENDPOINT || 'http://localhost:9000').replace(/\/$/, '');
|
||||
return `${base}/${bucketName}/${objectKey}`;
|
||||
export function buildMinioUrl(_bucketName: string, objectKey: string, _publicUrlOverride?: string | null): string {
|
||||
return `${getAppUrl()}/uploads/${objectKey}`;
|
||||
}
|
||||
|
||||
class MinioService {
|
||||
class LocalStorageService {
|
||||
async uploadMedia(
|
||||
file: Buffer | string,
|
||||
fileName: string,
|
||||
_userId: string,
|
||||
mimeType: string
|
||||
_mimeType: string
|
||||
): Promise<{
|
||||
publicId: string;
|
||||
originalUrl: string;
|
||||
@@ -35,78 +35,55 @@ class MinioService {
|
||||
instagramFeedUrl: string | null;
|
||||
instagramStoryUrl: string | null;
|
||||
}> {
|
||||
const config = await storage.getCloudinaryConfig();
|
||||
if (!config) throw new Error('MinIO non configuré. Veuillez configurer le stockage dans les Paramètres.');
|
||||
const dir = path.join(UPLOADS_BASE, 'media');
|
||||
ensureDir(dir);
|
||||
|
||||
const client = getS3Client(config.apiKey, config.apiSecret, config.endpointUrl);
|
||||
const bucket = config.cloudName;
|
||||
const safeName = path.basename(fileName).replace(/\s+/g, '-');
|
||||
const objectKey = `social-flow/${Date.now()}-${safeName}`;
|
||||
const safeName = path.basename(fileName).replace(/[^a-zA-Z0-9._-]/g, '-');
|
||||
const objectKey = `media/${Date.now()}-${safeName}`;
|
||||
const filePath = path.join(UPLOADS_BASE, objectKey);
|
||||
|
||||
const body: Buffer = typeof file === 'string' ? fs.readFileSync(file) : file;
|
||||
fs.writeFileSync(filePath, body);
|
||||
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: bucket,
|
||||
Key: objectKey,
|
||||
Body: body,
|
||||
ContentType: mimeType,
|
||||
}));
|
||||
|
||||
const url = buildMinioUrl(bucket, objectKey, config.publicUrl);
|
||||
const url = buildMinioUrl('', objectKey);
|
||||
return { publicId: objectKey, originalUrl: url, facebookFeedUrl: url, instagramFeedUrl: url, instagramStoryUrl: url };
|
||||
}
|
||||
|
||||
async deleteMedia(publicId: string, _userId: string, _mediaType: 'image' | 'video'): Promise<void> {
|
||||
const config = await storage.getCloudinaryConfig();
|
||||
if (!config) throw new Error('MinIO non configuré.');
|
||||
|
||||
const client = getS3Client(config.apiKey, config.apiSecret, config.endpointUrl);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: config.cloudName, Key: publicId }));
|
||||
const filePath = path.join(UPLOADS_BASE, publicId);
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
}
|
||||
|
||||
async uploadStoryImageWithText(buffer: Buffer, originalFileName: string): Promise<string> {
|
||||
const config = await storage.getCloudinaryConfig();
|
||||
if (!config) throw new Error('MinIO non configuré.');
|
||||
const dir = path.join(UPLOADS_BASE, 'stories');
|
||||
ensureDir(dir);
|
||||
|
||||
const client = getS3Client(config.apiKey, config.apiSecret, config.endpointUrl);
|
||||
const safeName = path.basename(originalFileName).replace(/\s+/g, '-');
|
||||
const objectKey = `social-flow/stories/story-${Date.now()}-${safeName}`;
|
||||
const safeName = path.basename(originalFileName).replace(/[^a-zA-Z0-9._-]/g, '-');
|
||||
const objectKey = `stories/story-${Date.now()}-${safeName}`;
|
||||
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: config.cloudName,
|
||||
Key: objectKey,
|
||||
Body: buffer,
|
||||
ContentType: 'image/jpeg',
|
||||
}));
|
||||
|
||||
return buildMinioUrl(config.cloudName, objectKey, config.publicUrl);
|
||||
fs.writeFileSync(path.join(UPLOADS_BASE, objectKey), buffer);
|
||||
return buildMinioUrl('', objectKey);
|
||||
}
|
||||
|
||||
async uploadLogo(buffer: Buffer, fileName: string): Promise<{ publicId: string; url: string }> {
|
||||
const config = await storage.getCloudinaryConfig();
|
||||
if (!config) throw new Error('MinIO non configuré.');
|
||||
const dir = path.join(UPLOADS_BASE, 'logos');
|
||||
ensureDir(dir);
|
||||
|
||||
const client = getS3Client(config.apiKey, config.apiSecret, config.endpointUrl);
|
||||
const ext = path.extname(fileName) || '.png';
|
||||
const objectKey = `social-flow/logos/logo-${Date.now()}${ext}`;
|
||||
const objectKey = `logos/logo-${Date.now()}${ext}`;
|
||||
|
||||
await client.send(new PutObjectCommand({
|
||||
Bucket: config.cloudName,
|
||||
Key: objectKey,
|
||||
Body: buffer,
|
||||
ContentType: fileName.match(/\.png$/i) ? 'image/png' : 'image/jpeg',
|
||||
}));
|
||||
|
||||
return { publicId: objectKey, url: buildMinioUrl(config.cloudName, objectKey, config.publicUrl) };
|
||||
fs.writeFileSync(path.join(UPLOADS_BASE, objectKey), buffer);
|
||||
return { publicId: objectKey, url: buildMinioUrl('', objectKey) };
|
||||
}
|
||||
|
||||
async deleteLogo(publicId: string): Promise<void> {
|
||||
const config = await storage.getCloudinaryConfig();
|
||||
if (!config) throw new Error('MinIO non configuré.');
|
||||
|
||||
const client = getS3Client(config.apiKey, config.apiSecret, config.endpointUrl);
|
||||
await client.send(new DeleteObjectCommand({ Bucket: config.cloudName, Key: publicId }));
|
||||
const filePath = path.join(UPLOADS_BASE, publicId);
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.unlinkSync(filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const minioService = new MinioService();
|
||||
export const minioService = new LocalStorageService();
|
||||
@@ -1,6 +1,7 @@
|
||||
import cron, { ScheduledTask } from "node-cron";
|
||||
import { storage } from "../storage";
|
||||
import { facebookService } from "./facebook";
|
||||
import { minioService as storageService } from "./minio";
|
||||
|
||||
export class SchedulerService {
|
||||
private task: ScheduledTask | null = null;
|
||||
@@ -124,6 +125,18 @@ export class SchedulerService {
|
||||
status: "published",
|
||||
});
|
||||
|
||||
// Delete local video files immediately after successful publishing
|
||||
for (const mediaItem of mediaList) {
|
||||
if (mediaItem.type === 'video' && mediaItem.cloudinaryPublicId) {
|
||||
try {
|
||||
await storageService.deleteMedia(mediaItem.cloudinaryPublicId, mediaItem.userId, 'video');
|
||||
console.log(`[Scheduler] Deleted local video file for media ${mediaItem.id}`);
|
||||
} catch (err) {
|
||||
console.warn(`[Scheduler] Failed to delete local video file for media ${mediaItem.id}:`, err);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Successfully published post ${post.id} with external ID: ${externalPostId}`);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user