mirror of
https://github.com/R0m1k3/Apiflow-gateway.git
synced 2026-10-19 05:07:31 +02:00
feat: gateway sécurisé, miroir de schéma automatique et sync incrémentale
- Schéma Hostinger recopié automatiquement depuis PG Debian (colonnes, PK, index) : plus d'init.sql maintenu à la main (corrige articles.artcentrale, mvtart.no_id et les 12 tables manquantes) - Stratégies : full transactionnel (DELETE + INSERT, garde-fou de volume) et window (signature mensuelle + 10 derniers jours) pour les grosses tables - Attente de la fin de la sync Debian (_run), pg_try_advisory_lock, cron 07:30 Europe/Paris, force honoré, alertes webhook - Dashboard : basic auth, CSP sans script inline, plus d'innerHTML, port 3002 non publié (accès via Nginx Proxy Manager) - Secrets hors du compose (.env), frps épinglé 0.71.0 avec TLS forcé, allowPorts et additionalScopes - Suppression de sync.sh, tables.js, utils.js, init.sql ; README de déploiement L'ancien mot de passe de la base `post` figure dans l'historique git : il doit être changé sur le serveur (voir README). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f8575b9ff2
commit
ea084034b9
26 files changed
+2767
-922
No files matched your search
+103
-248
@@ -1,281 +1,136 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Dashboard de la sync Hostinger (port 3002, joint par Nginx Proxy Manager sur nginx_default).
|
||||
*
|
||||
* Sécurité :
|
||||
* - HTTP Basic auth (DASHBOARD_USER / DASHBOARD_PASSWORD), comparaison à temps constant ;
|
||||
* sans ces variables, tout répond 503 sauf GET /healthz (healthcheck Docker).
|
||||
* - POST /api/sync/run exige l'en-tête X-Requested-By (bloque les formulaires cross-site / CSRF).
|
||||
* - Aucune donnée dynamique insérée via innerHTML ; CSP sans script inline.
|
||||
*/
|
||||
const crypto = require('node:crypto');
|
||||
const path = require('node:path');
|
||||
const express = require('express');
|
||||
const { config } = require('./config');
|
||||
const log = require('./logger');
|
||||
const { getDst } = require('./db');
|
||||
const { TABLES } = require('./tables');
|
||||
const { state } = require('./sync');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.SYNC_DASHBOARD_PORT || 3002;
|
||||
const app = express();
|
||||
app.disable('x-powered-by');
|
||||
app.set('trust proxy', 'loopback, uniquelocal');
|
||||
|
||||
const TABLES_ORDER = TABLES.map(t => t.name);
|
||||
const TABLES_TOTAL = TABLES_ORDER.length;
|
||||
let runFn = null;
|
||||
let nextRunFn = () => null;
|
||||
function registerSyncFn(fn) { runFn = fn; }
|
||||
function registerNextRunFn(fn) { nextRunFn = fn; }
|
||||
|
||||
const state = {
|
||||
running: false,
|
||||
progress: null,
|
||||
};
|
||||
app.get('/healthz', (req, res) => res.json({ ok: true }));
|
||||
|
||||
const logBuffer = [];
|
||||
function pushLog(msg, level = 'info') {
|
||||
logBuffer.push({ ts: new Date().toISOString(), msg, level });
|
||||
if (logBuffer.length > 300) logBuffer.shift();
|
||||
const sha = s => crypto.createHash('sha256').update(String(s)).digest();
|
||||
function safeEqual(a, b) {
|
||||
return crypto.timingSafeEqual(sha(a), sha(b));
|
||||
}
|
||||
|
||||
const _log = console.log.bind(console);
|
||||
const _err = console.error.bind(console);
|
||||
console.log = (...a) => { pushLog(a.join(' '), 'info'); _log(...a); };
|
||||
console.error = (...a) => { pushLog(a.join(' '), 'error'); _err(...a); };
|
||||
|
||||
function setProgress(tableName, index) {
|
||||
state.progress = {
|
||||
current: tableName,
|
||||
index,
|
||||
total: TABLES_TOTAL,
|
||||
pct: Math.round((index / TABLES_TOTAL) * 100),
|
||||
startedAt: state.progress?.startedAt || new Date().toISOString(),
|
||||
};
|
||||
function basicAuth(req, res, next) {
|
||||
if (!config.dashboardUser || !config.dashboardPass) {
|
||||
return res.status(503).type('text/plain; charset=utf-8')
|
||||
.send('dashboard désactivé: définir DASHBOARD_USER/DASHBOARD_PASSWORD');
|
||||
}
|
||||
const header = req.get('authorization') || '';
|
||||
const m = /^Basic\s+([A-Za-z0-9+/=]+)$/i.exec(header);
|
||||
let user = '';
|
||||
let pass = '';
|
||||
if (m) {
|
||||
const decoded = Buffer.from(m[1], 'base64').toString('utf8');
|
||||
const i = decoded.indexOf(':');
|
||||
if (i >= 0) { user = decoded.slice(0, i); pass = decoded.slice(i + 1); }
|
||||
}
|
||||
// Les deux comparaisons sont toujours évaluées (pas de court-circuit)
|
||||
const okUser = safeEqual(user, config.dashboardUser);
|
||||
const okPass = safeEqual(pass, config.dashboardPass);
|
||||
if (m && okUser && okPass) return next();
|
||||
if (m) log.warn(`Authentification dashboard refusée (ip ${req.ip})`);
|
||||
res.set('WWW-Authenticate', 'Basic realm="ff_sync_hostinger", charset="UTF-8"');
|
||||
return res.status(401).type('text/plain; charset=utf-8').send('Authentification requise');
|
||||
}
|
||||
function clearProgress() { state.progress = null; }
|
||||
|
||||
let syncAllFn = null;
|
||||
function registerSyncFn(fn) { syncAllFn = fn; }
|
||||
function setSyncRunning(val) { state.running = val; }
|
||||
function isSyncRunning() { return state.running; }
|
||||
|
||||
app.use(express.json());
|
||||
app.use(basicAuth);
|
||||
app.use((req, res, next) => {
|
||||
res.set({
|
||||
'Content-Security-Policy': "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'",
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'Referrer-Policy': 'no-referrer',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
next();
|
||||
});
|
||||
app.use(express.json({ limit: '10kb' }));
|
||||
|
||||
app.get('/api/status', async (req, res) => {
|
||||
let tables = [];
|
||||
let dbError = null;
|
||||
try {
|
||||
const dst = getDst();
|
||||
const { rows } = await dst.query(
|
||||
`SELECT table_name, last_sync, rows_synced, status, error_msg
|
||||
FROM sync_log ORDER BY table_name`
|
||||
const { rows } = await getDst().query(
|
||||
`SELECT table_name, last_sync, rows_synced, status, error_msg, strategy, duration_ms, detail
|
||||
FROM public.sync_log ORDER BY table_name`,
|
||||
);
|
||||
res.json({ running: state.running, progress: state.progress, tables: rows });
|
||||
tables = rows;
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
// 42P01 = sync_log pas encore créée (aucune exécution)
|
||||
if (err.code !== '42P01') dbError = 'base destination injoignable';
|
||||
if (err.code !== '42P01') log.error(`/api/status : ${err.message}`);
|
||||
}
|
||||
res.json({
|
||||
running: state.running,
|
||||
force: state.force,
|
||||
phase: state.phase,
|
||||
progress: state.progress,
|
||||
lastRun: state.lastRun,
|
||||
tables,
|
||||
dbError,
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/logs', (req, res) => {
|
||||
const limit = parseInt(req.query.limit) || 150;
|
||||
res.json(logBuffer.slice(-limit));
|
||||
res.json(log.getLogs(parseInt(req.query.limit, 10) || 200));
|
||||
});
|
||||
|
||||
app.get('/api/next', (req, res) => {
|
||||
const now = new Date();
|
||||
const next = new Date(now);
|
||||
next.setHours(5, 30, 0, 0);
|
||||
if (next <= now) next.setDate(next.getDate() + 1);
|
||||
res.json({ next: next.toISOString() });
|
||||
const next = nextRunFn();
|
||||
res.json({ next: next ? next.toISOString() : null, cron: config.cron, timeZone: config.timeZone });
|
||||
});
|
||||
|
||||
app.post('/api/sync/run', async (req, res) => {
|
||||
app.post('/api/sync/run', (req, res) => {
|
||||
if (req.get('x-requested-by') !== 'ff-dashboard') {
|
||||
return res.status(400).json({ error: 'en-tête X-Requested-By manquant' });
|
||||
}
|
||||
if (state.running) return res.status(409).json({ error: 'Sync déjà en cours' });
|
||||
if (!syncAllFn) return res.status(500).json({ error: 'syncAll non enregistré' });
|
||||
res.json({ message: 'Sync lancée' });
|
||||
syncAllFn(true).catch(err => console.error('Erreur sync manuelle:', err.message));
|
||||
if (!runFn) return res.status(500).json({ error: 'sync non initialisée' });
|
||||
const force = req.body?.force === true;
|
||||
log.info(`Sync ${force ? 'FORCÉE ' : ''}demandée depuis le dashboard (ip ${req.ip})`);
|
||||
res.status(202).json({ message: force ? 'Sync forcée lancée' : 'Sync lancée' });
|
||||
runFn({ force, trigger: 'dashboard' }).catch(err => log.error(`Sync manuelle : ${err.message}`));
|
||||
});
|
||||
|
||||
app.get('/', (req, res) => res.send(HTML));
|
||||
const PUBLIC_DIR = path.join(__dirname, 'public');
|
||||
app.get('/', (req, res) => res.sendFile(path.join(PUBLIC_DIR, 'index.html')));
|
||||
app.get('/app.js', (req, res) => res.type('application/javascript').sendFile(path.join(PUBLIC_DIR, 'app.js')));
|
||||
|
||||
app.use((req, res) => res.status(404).json({ error: 'not found' }));
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
app.use((err, req, res, next) => {
|
||||
log.error(`HTTP ${req.method} ${req.path} : ${err.message}`);
|
||||
res.status(500).json({ error: 'erreur interne' });
|
||||
});
|
||||
|
||||
function startDashboard() {
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`Dashboard sync Hostinger : http://0.0.0.0:${PORT}`);
|
||||
if (!config.dashboardUser || !config.dashboardPass) {
|
||||
log.warn('DASHBOARD_USER / DASHBOARD_PASSWORD non définis : dashboard et API désactivés (503)');
|
||||
}
|
||||
return app.listen(config.port, '0.0.0.0', () => {
|
||||
log.info(`Dashboard sync Hostinger : port ${config.port}`);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { startDashboard, registerSyncFn, setSyncRunning, isSyncRunning, setProgress, clearProgress };
|
||||
|
||||
const HTML = `<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Sync Hostinger</title>
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:system-ui,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh}
|
||||
header{background:#1e293b;padding:1rem 2rem;display:flex;align-items:center;gap:1rem;border-bottom:1px solid #334155}
|
||||
header h1{font-size:1.2rem;font-weight:700;color:#f8fafc}
|
||||
.subtitle{font-size:.75rem;color:#64748b}
|
||||
.badge{font-size:.75rem;padding:.25rem .6rem;border-radius:9999px;font-weight:600}
|
||||
.badge.idle{background:#1e3a5f;color:#93c5fd}
|
||||
.badge.running{background:#713f12;color:#fde68a;animation:pulse 1s infinite}
|
||||
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.5}}
|
||||
main{padding:1.5rem 2rem;max-width:1100px;margin:0 auto}
|
||||
.actions{display:flex;align-items:center;gap:1rem;flex-wrap:wrap;margin-bottom:1.5rem}
|
||||
button{padding:.55rem 1.3rem;border:none;border-radius:.5rem;cursor:pointer;font-weight:600;font-size:.85rem;transition:.15s}
|
||||
#btnSync{background:#2563eb;color:#fff}
|
||||
#btnSync:hover:not(:disabled){background:#1d4ed8}
|
||||
#btnSync:disabled{opacity:.4;cursor:not-allowed}
|
||||
#btnRefresh{background:#334155;color:#cbd5e1}
|
||||
#btnRefresh:hover{background:#475569}
|
||||
#nextSync{font-size:.8rem;color:#64748b;margin-left:auto}
|
||||
#msg{font-size:.82rem;padding:.35rem .75rem;border-radius:.4rem;display:none}
|
||||
#msg.ok{background:#14532d;color:#86efac;display:inline-block}
|
||||
#msg.err{background:#450a0a;color:#fca5a5;display:inline-block}
|
||||
#progressBox{background:#1e293b;border:1px solid #334155;border-radius:.75rem;padding:1rem 1.25rem;margin-bottom:1.5rem;display:none}
|
||||
#progressBox.visible{display:block}
|
||||
.progress-header{display:flex;justify-content:space-between;margin-bottom:.6rem;font-size:.85rem}
|
||||
.progress-bar-bg{background:#0f172a;border-radius:9999px;height:12px;overflow:hidden}
|
||||
.progress-bar-fill{height:100%;background:linear-gradient(90deg,#059669,#2563eb);border-radius:9999px;transition:width .5s ease}
|
||||
.progress-detail{margin-top:.5rem;font-size:.78rem;color:#64748b}
|
||||
.grid{display:grid;grid-template-columns:1fr 1fr;gap:1.5rem}
|
||||
@media(max-width:700px){.grid{grid-template-columns:1fr}}
|
||||
.card{background:#1e293b;border:1px solid #334155;border-radius:.75rem;overflow:hidden}
|
||||
.card-header{padding:.65rem 1rem;background:#0f172a;border-bottom:1px solid #334155;font-size:.75rem;font-weight:600;text-transform:uppercase;letter-spacing:.05em;color:#94a3b8}
|
||||
table{width:100%;border-collapse:collapse;font-size:.85rem}
|
||||
th{padding:.55rem 1rem;text-align:left;color:#64748b;font-weight:500;border-bottom:1px solid #334155}
|
||||
td{padding:.55rem 1rem;border-bottom:1px solid #1e293b}
|
||||
tr:last-child td{border-bottom:none}
|
||||
.ok{color:#4ade80}.err{color:#f87171}
|
||||
.log-box{height:320px;overflow-y:auto;font-family:monospace;font-size:.76rem;padding:.75rem 1rem;background:#0a0f1e;color:#a5f3fc;line-height:1.7;user-select:text;cursor:text}
|
||||
.log-line{white-space:pre-wrap;word-break:break-all}
|
||||
.log-line.error{color:#f87171}
|
||||
.ts{color:#475569;margin-right:.5rem}
|
||||
#btnCopyLogs{background:#334155;color:#cbd5e1;font-size:.75rem;padding:.35rem .8rem}
|
||||
#btnCopyLogs:hover{background:#475569}
|
||||
#lastRefresh{font-size:.75rem;color:#475569}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>⚡ Sync Hostinger</h1>
|
||||
<span class="subtitle">PG Debian → PG Hostinger</span>
|
||||
<span id="statusBadge" class="badge idle">Idle</span>
|
||||
<span id="lastRefresh" style="margin-left:auto"></span>
|
||||
</header>
|
||||
<main>
|
||||
<div class="actions">
|
||||
<button id="btnSync" onclick="runSync()">▶ Lancer sync maintenant</button>
|
||||
<button id="btnRefresh" onclick="refresh()">↺ Rafraîchir</button>
|
||||
<span id="msg"></span>
|
||||
<span id="nextSync"></span>
|
||||
</div>
|
||||
|
||||
<div id="progressBox">
|
||||
<div class="progress-header">
|
||||
<span id="progressLabel">Synchronisation en cours…</span>
|
||||
<span id="progressPct" style="font-weight:700;color:#93c5fd">0%</span>
|
||||
</div>
|
||||
<div class="progress-bar-bg">
|
||||
<div class="progress-bar-fill" id="progressBar" style="width:0%"></div>
|
||||
</div>
|
||||
<div class="progress-detail" id="progressDetail"></div>
|
||||
</div>
|
||||
|
||||
<div class="grid">
|
||||
<div class="card" style="grid-column:1/-1">
|
||||
<div class="card-header">Statut des tables</div>
|
||||
<table>
|
||||
<thead><tr><th>Table</th><th>Dernière sync</th><th>Lignes</th><th>Statut</th></tr></thead>
|
||||
<tbody id="tableBody"><tr><td colspan="4" style="color:#475569;padding:1rem">Chargement…</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="card" style="grid-column:1/-1">
|
||||
<div class="card-header" style="display:flex;justify-content:space-between;align-items:center">
|
||||
<span>Logs</span>
|
||||
<button id="btnCopyLogs" onclick="copyLogs()">Copier les logs</button>
|
||||
</div>
|
||||
<div class="log-box" id="logBox">Chargement…</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
<script>
|
||||
function fmtDate(iso){
|
||||
if(!iso) return '—';
|
||||
return new Date(iso).toLocaleString('fr-FR');
|
||||
}
|
||||
function renderProgress(progress, running){
|
||||
const box = document.getElementById('progressBox');
|
||||
if(!running || !progress){ box.classList.remove('visible'); return; }
|
||||
box.classList.add('visible');
|
||||
document.getElementById('progressBar').style.width = progress.pct + '%';
|
||||
document.getElementById('progressPct').textContent = progress.pct + '%';
|
||||
document.getElementById('progressLabel').textContent =
|
||||
'Sync en cours — table ' + progress.index + '/' + progress.total;
|
||||
const elapsed = Math.round((Date.now() - new Date(progress.startedAt)) / 1000);
|
||||
document.getElementById('progressDetail').textContent =
|
||||
'En cours : ' + progress.current + ' | Durée : ' + elapsed + 's';
|
||||
}
|
||||
function renderStatus(data){
|
||||
const badge = document.getElementById('statusBadge');
|
||||
badge.textContent = data.running ? 'En cours…' : 'Idle';
|
||||
badge.className = 'badge ' + (data.running ? 'running' : 'idle');
|
||||
document.getElementById('btnSync').disabled = data.running;
|
||||
renderProgress(data.progress, data.running);
|
||||
if(!data.tables.length){
|
||||
document.getElementById('tableBody').innerHTML =
|
||||
'<tr><td colspan="4" style="color:#475569;padding:1rem">Aucune sync — cliquez sur Lancer sync maintenant</td></tr>';
|
||||
return;
|
||||
}
|
||||
document.getElementById('tableBody').innerHTML = data.tables.map(t => \`
|
||||
<tr>
|
||||
<td><strong>\${t.table_name}</strong></td>
|
||||
<td>\${fmtDate(t.last_sync)}</td>
|
||||
<td>\${t.rows_synced ?? '—'}</td>
|
||||
<td class="\${t.status==='ok'?'ok':'err'}">\${t.status ?? '—'}\${t.error_msg?' — '+t.error_msg:''}</td>
|
||||
</tr>
|
||||
\`).join('');
|
||||
}
|
||||
function renderLogs(logs){
|
||||
const box = document.getElementById('logBox');
|
||||
if(!logs.length){ box.textContent = 'Aucun log.'; return; }
|
||||
const atBottom = box.scrollTop + box.clientHeight >= box.scrollHeight - 10;
|
||||
box.innerHTML = logs.map(l => \`
|
||||
<div class="log-line \${l.level}">
|
||||
<span class="ts">\${new Date(l.ts).toLocaleTimeString('fr-FR')}</span>\${l.msg}
|
||||
</div>
|
||||
\`).join('');
|
||||
if(atBottom) box.scrollTop = box.scrollHeight;
|
||||
}
|
||||
async function refresh(){
|
||||
try{
|
||||
const [s, logs, next] = await Promise.all([
|
||||
fetch('/api/status').then(r=>r.json()),
|
||||
fetch('/api/logs?limit=150').then(r=>r.json()),
|
||||
fetch('/api/next').then(r=>r.json()),
|
||||
]);
|
||||
renderStatus(s);
|
||||
renderLogs(logs);
|
||||
document.getElementById('nextSync').textContent = 'Prochaine sync auto : ' + fmtDate(next.next);
|
||||
document.getElementById('lastRefresh').textContent = 'Mis à jour ' + new Date().toLocaleTimeString('fr-FR');
|
||||
} catch(e){ console.error(e); }
|
||||
}
|
||||
function copyLogs(){
|
||||
const box = document.getElementById('logBox');
|
||||
const text = Array.from(box.querySelectorAll('.log-line')).map(l => l.textContent).join('\\n');
|
||||
const btn = document.getElementById('btnCopyLogs');
|
||||
function onSuccess(){ btn.textContent = 'Copié !'; setTimeout(() => { btn.textContent = 'Copier les logs'; }, 2000); }
|
||||
if(navigator.clipboard && window.isSecureContext){
|
||||
navigator.clipboard.writeText(text).then(onSuccess).catch(fallback);
|
||||
} else { fallback(); }
|
||||
function fallback(){
|
||||
const ta = document.createElement('textarea');
|
||||
ta.value = text;
|
||||
ta.style.cssText = 'position:fixed;top:-9999px;left:-9999px;opacity:0';
|
||||
document.body.appendChild(ta); ta.focus(); ta.select();
|
||||
try{ document.execCommand('copy'); onSuccess(); } catch(e){ alert('Sélectionnez les logs et Ctrl+C'); }
|
||||
document.body.removeChild(ta);
|
||||
}
|
||||
}
|
||||
async function runSync(){
|
||||
const msg = document.getElementById('msg');
|
||||
msg.style.display = 'none';
|
||||
const r = await fetch('/api/sync/run',{method:'POST'});
|
||||
const data = await r.json();
|
||||
msg.textContent = r.ok ? '✓ ' + data.message : '✗ ' + data.error;
|
||||
msg.className = r.ok ? 'ok' : 'err';
|
||||
setTimeout(()=>{ msg.style.display='none'; }, 5000);
|
||||
setTimeout(refresh, 800);
|
||||
}
|
||||
let timer;
|
||||
async function loop(){
|
||||
await refresh();
|
||||
const s = await fetch('/api/status').then(r=>r.json()).catch(()=>({}));
|
||||
timer = setTimeout(loop, s.running ? 3000 : 15000);
|
||||
}
|
||||
loop();
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
module.exports = { app, startDashboard, registerSyncFn, registerNextRunFn };
|
||||
Reference in new issue
Block a user