mirror of
https://github.com/R0m1k3/Apiflow-gateway.git
synced 2026-10-20 13:38:12 +02:00
feat: gateway sécurisé, miroir de schéma automatique et sync incrémentale
- Schéma Hostinger recopié automatiquement depuis PG Debian (colonnes, PK, index) : plus d'init.sql maintenu à la main (corrige articles.artcentrale, mvtart.no_id et les 12 tables manquantes) - Stratégies : full transactionnel (DELETE + INSERT, garde-fou de volume) et window (signature mensuelle + 10 derniers jours) pour les grosses tables - Attente de la fin de la sync Debian (_run), pg_try_advisory_lock, cron 07:30 Europe/Paris, force honoré, alertes webhook - Dashboard : basic auth, CSP sans script inline, plus d'innerHTML, port 3002 non publié (accès via Nginx Proxy Manager) - Secrets hors du compose (.env), frps épinglé 0.71.0 avec TLS forcé, allowPorts et additionalScopes - Suppression de sync.sh, tables.js, utils.js, init.sql ; README de déploiement L'ancien mot de passe de la base `post` figure dans l'historique git : il doit être changé sur le serveur (voir README). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
MichaelandClaude Opus 5.5 committed
1 parent
f8575b9ff2
commit
ea084034b9
26 files changed
+2767
-922
No files matched your search
@@ -0,0 +1,5 @@
|
||||
node_modules
|
||||
npm-debug.log
|
||||
test
|
||||
.env
|
||||
*.md
|
||||
+13
-4
@@ -1,12 +1,21 @@
|
||||
FROM node:20-alpine
|
||||
# Service ff_sync_hostinger — miroir PG Debian → PG Hostinger
|
||||
FROM node:22-alpine
|
||||
|
||||
# tzdata : fuseau Europe/Paris pour l'heure locale du process (TZ défini dans docker-compose)
|
||||
RUN apk add --no-cache tzdata
|
||||
|
||||
WORKDIR /app
|
||||
ENV NODE_ENV=production
|
||||
|
||||
COPY package.json ./
|
||||
RUN npm install --omit=dev
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci --omit=dev && npm cache clean --force
|
||||
|
||||
COPY . .
|
||||
COPY --chown=node:node . .
|
||||
|
||||
USER node
|
||||
EXPOSE 3002
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD node -e "fetch('http://127.0.0.1:'+(process.env.SYNC_DASHBOARD_PORT||3002)+'/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))"
|
||||
|
||||
CMD ["node", "index.js"]
|
||||
@@ -0,0 +1,32 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Alerte par webhook (Slack, Discord, Mattermost, ntfy… compatibles JSON).
|
||||
* Envoie {text, content} : Slack/Mattermost lisent "text", Discord lit "content".
|
||||
* Ne lève jamais d'exception.
|
||||
*/
|
||||
const { config } = require('./config');
|
||||
const log = require('./logger');
|
||||
|
||||
async function sendAlert(message) {
|
||||
if (!config.alertWebhookUrl) return false;
|
||||
const text = String(message).slice(0, 1900); // Discord : 2000 caractères max
|
||||
try {
|
||||
const res = await fetch(config.alertWebhookUrl, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text, content: text }),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
if (!res.ok) {
|
||||
log.warn(`Alerte webhook : réponse HTTP ${res.status}`);
|
||||
return false;
|
||||
}
|
||||
log.info('Alerte envoyée');
|
||||
return true;
|
||||
} catch (err) {
|
||||
log.warn(`Alerte webhook impossible : ${err.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { sendAlert };
|
||||
@@ -0,0 +1,59 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Configuration du service, lue depuis les variables d'environnement
|
||||
* (fournies par docker-compose depuis gateway/.env).
|
||||
*/
|
||||
const { parseList, parseWindowTables } = require('./sql');
|
||||
|
||||
/**
|
||||
* Tables copiées par "fenêtre" : signature par mois + rechargement des mois différents
|
||||
* + rechargement systématique des N derniers jours. Clé = table source, valeur = colonne date.
|
||||
* Une table configurée mais absente de la source est ignorée silencieusement.
|
||||
* Surcharge possible via WINDOW_TABLES="table:colonne,table:colonne".
|
||||
*/
|
||||
const DEFAULT_WINDOW_TABLES = {
|
||||
mvtart: 'datmvt',
|
||||
mvtreg: 'datmvt',
|
||||
mvtart_motif: 'datmvt',
|
||||
histo_art_gamme: 'date',
|
||||
demarques: 'datmvt',
|
||||
meteo: 'date',
|
||||
fidelite_jour: 'date',
|
||||
};
|
||||
|
||||
function int(name, def) {
|
||||
const v = parseInt(process.env[name], 10);
|
||||
return Number.isFinite(v) && v >= 0 ? v : def;
|
||||
}
|
||||
|
||||
const config = {
|
||||
cron: process.env.SYNC_CRON || '30 7 * * *',
|
||||
timeZone: process.env.SYNC_TZ || 'Europe/Paris',
|
||||
port: int('SYNC_DASHBOARD_PORT', 3002),
|
||||
|
||||
// Tables source exclues (en plus de sync_log, traitée à part → source_sync_log)
|
||||
excludeTables: new Set(parseList(process.env.EXCLUDE_TABLES)),
|
||||
windowTables: process.env.WINDOW_TABLES
|
||||
? parseWindowTables(process.env.WINDOW_TABLES)
|
||||
: DEFAULT_WINDOW_TABLES,
|
||||
recentDays: int('WINDOW_RECENT_DAYS', 10),
|
||||
|
||||
// Attente de la fin de la sync Debian (ligne sync_log '_run' = 'running')
|
||||
waitPollMs: int('DEBIAN_WAIT_POLL_MIN', 2) * 60 * 1000,
|
||||
waitMaxMs: int('DEBIAN_WAIT_MAX_MIN', 180) * 60 * 1000,
|
||||
|
||||
// Garde-fou volume : source < 50 % de la destination (si destination > 100 lignes) → table ignorée
|
||||
volumeMinRatio: 0.5,
|
||||
volumeMinRows: 100,
|
||||
|
||||
fetchSize: 5000,
|
||||
|
||||
dashboardUser: process.env.DASHBOARD_USER || '',
|
||||
dashboardPass: process.env.DASHBOARD_PASSWORD || '',
|
||||
alertWebhookUrl: process.env.ALERT_WEBHOOK_URL || '',
|
||||
|
||||
// Clé du verrou consultatif (pg_try_advisory_lock) côté destination — constante
|
||||
lockKey: 4646535901,
|
||||
};
|
||||
|
||||
module.exports = { config, DEFAULT_WINDOW_TABLES };
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Stratégies de copie des données source → destination.
|
||||
*
|
||||
* - full : BEGIN; DELETE FROM; INSERT de toutes les lignes; COMMIT (une seule transaction
|
||||
* destination : les lecteurs ne voient jamais une table vide ou partielle).
|
||||
* - window : signature (nb lignes + somme des hash) par mois des deux côtés, puis pour
|
||||
* chaque mois différent : BEGIN; DELETE du mois; INSERT depuis la source; COMMIT.
|
||||
* Les N derniers jours sont toujours rechargés.
|
||||
*
|
||||
* Lecture source par curseur serveur (FETCH par lots) : jamais une table entière en RAM.
|
||||
* Les valeurs transitent en texte brut (pas de conversion JS) : aucune perte de précision
|
||||
* sur les timestamps, numeric, bigint, bytea, tableaux…
|
||||
*/
|
||||
const { config } = require('./config');
|
||||
const log = require('./logger');
|
||||
const {
|
||||
qt, buildSelect, buildInsert, insertChunkSize,
|
||||
buildSignatureQuery, signatureMap, signatureTotal, diffMonths,
|
||||
monthCondition, recentCondition, cutoffDate,
|
||||
} = require('./sql');
|
||||
|
||||
// Désactive tous les parseurs de types de node-postgres : chaque valeur reste la chaîne PG
|
||||
const RAW_TYPES = { getTypeParser: () => v => v };
|
||||
|
||||
let cursorSeq = 0;
|
||||
|
||||
/**
|
||||
* Copie les lignes d'un SELECT source vers une table destination.
|
||||
* src doit être dans une transaction (DECLARE CURSOR l'exige).
|
||||
*/
|
||||
async function streamRows(src, dst, srcTable, dstTable, cols, where) {
|
||||
const cursor = `ff_cur_${++cursorSeq}`;
|
||||
const chunk = insertChunkSize(cols.length);
|
||||
await src.query(`DECLARE ${cursor} NO SCROLL CURSOR FOR ${buildSelect(srcTable, cols, where)}`);
|
||||
let total = 0;
|
||||
let insertSql = null;
|
||||
let insertRows = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const res = await src.query({ text: `FETCH ${config.fetchSize} FROM ${cursor}`, rowMode: 'array', types: RAW_TYPES });
|
||||
const rows = res.rows;
|
||||
if (!rows.length) break;
|
||||
for (let i = 0; i < rows.length; i += chunk) {
|
||||
const part = rows.slice(i, i + chunk);
|
||||
if (part.length !== insertRows) {
|
||||
insertSql = buildInsert(dstTable, cols, part.length);
|
||||
insertRows = part.length;
|
||||
}
|
||||
await dst.query(insertSql, part.flat());
|
||||
}
|
||||
total += rows.length;
|
||||
}
|
||||
} finally {
|
||||
await src.query(`CLOSE ${cursor}`).catch(() => {});
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
async function count(client, table) {
|
||||
const { rows } = await client.query(`SELECT count(*)::bigint AS n FROM ${qt(table)}`);
|
||||
return Number(rows[0].n);
|
||||
}
|
||||
|
||||
function volumeGuard(srcCount, dstCount, force) {
|
||||
if (force) return null;
|
||||
if (dstCount > config.volumeMinRows && srcCount < dstCount * config.volumeMinRatio) {
|
||||
return `garde-fou volume : source ${srcCount} lignes < 50 % de la destination (${dstCount}) — table non copiée (relancer en mode forcé si c'est voulu)`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exécute fn dans une transaction destination (ROLLBACK en cas d'erreur). */
|
||||
async function inDstTx(dst, fn) {
|
||||
await dst.query('BEGIN');
|
||||
try {
|
||||
// Ne pas rester bloqué indéfiniment derrière un lecteur long (et bloquer les suivants)
|
||||
await dst.query("SET LOCAL lock_timeout = '60s'");
|
||||
const r = await fn();
|
||||
await dst.query('COMMIT');
|
||||
return r;
|
||||
} catch (err) {
|
||||
await dst.query('ROLLBACK').catch(() => {});
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Copie complète transactionnelle.
|
||||
* Retourne { rows, detail } ou lève une erreur (dont le garde-fou volume).
|
||||
*/
|
||||
async function copyFull(src, dst, srcTable, dstTable, cols, force) {
|
||||
const srcCount = await count(src, srcTable);
|
||||
const dstCount = await count(dst, dstTable);
|
||||
const guard = volumeGuard(srcCount, dstCount, force);
|
||||
if (guard) throw new Error(guard);
|
||||
const rows = await inDstTx(dst, async () => {
|
||||
// DELETE (et non TRUNCATE) : les lecteurs continuent de voir l'ancienne version (MVCC)
|
||||
// au lieu d'être bloqués par un verrou ACCESS EXCLUSIVE ; l'autovacuum récupère l'espace.
|
||||
await dst.query(`DELETE FROM ${qt(dstTable)}`);
|
||||
return streamRows(src, dst, srcTable, dstTable, cols, null);
|
||||
});
|
||||
return { rows, detail: `copie complète (${dstCount} -> ${rows} lignes)` };
|
||||
}
|
||||
|
||||
/** Recharge les lignes vérifiant `where` (DELETE + INSERT dans une transaction). */
|
||||
async function reloadSlice(src, dst, table, cols, where) {
|
||||
return inDstTx(dst, async () => {
|
||||
await dst.query(`DELETE FROM ${qt(table)} WHERE ${where}`);
|
||||
return streamRows(src, dst, table, table, cols, where);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Copie par fenêtre de dates (tables de mouvements volumineuses).
|
||||
* Retourne { rows, detail }.
|
||||
*/
|
||||
async function copyWindow(src, dst, table, cols, dateCol, force) {
|
||||
if (force) {
|
||||
const r = await copyFull(src, dst, table, table, cols, true);
|
||||
return { rows: r.rows, detail: `mode forcé : ${r.detail}` };
|
||||
}
|
||||
const sigSql = buildSignatureQuery(table, cols, dateCol);
|
||||
const srcSig = signatureMap((await src.query(sigSql)).rows);
|
||||
let dstSig = signatureMap((await dst.query(sigSql)).rows);
|
||||
|
||||
const guard = volumeGuard(signatureTotal(srcSig), signatureTotal(dstSig), false);
|
||||
if (guard) throw new Error(guard);
|
||||
|
||||
// 1. Fenêtre glissante : les N derniers jours sont toujours rechargés
|
||||
const cutoff = cutoffDate(config.recentDays, config.timeZone);
|
||||
let rows = await reloadSlice(src, dst, table, cols, recentCondition(dateCol, cutoff));
|
||||
const recentRows = rows;
|
||||
|
||||
// 2. Mois dont la signature diffère (nouvelles colonnes, corrections, suppressions…)
|
||||
dstSig = signatureMap((await dst.query(sigSql)).rows);
|
||||
const months = diffMonths(srcSig, dstSig);
|
||||
for (const month of months) {
|
||||
const n = await reloadSlice(src, dst, table, cols, monthCondition(dateCol, month));
|
||||
log.info(`[${table}] mois ${month} rechargé (${n} lignes)`);
|
||||
rows += n;
|
||||
}
|
||||
return {
|
||||
rows,
|
||||
detail: `${recentRows} lignes depuis le ${cutoff} + ${months.length} mois rechargé(s)` +
|
||||
(months.length ? ` (${months.slice(0, 6).join(', ')}${months.length > 6 ? '…' : ''})` : ''),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { copyFull, copyWindow, streamRows, volumeGuard, RAW_TYPES };
|
||||
+44
-24
@@ -1,41 +1,61 @@
|
||||
'use strict';
|
||||
const { Pool } = require('pg');
|
||||
|
||||
/**
|
||||
* Paramètres de session identiques des deux côtés : indispensables pour que
|
||||
* ROW(...)::text (signatures par mois) produise le même texte sur la source et la
|
||||
* destination (fuseau, format des dates/intervalles, précision des flottants).
|
||||
*/
|
||||
const SESSION_OPTIONS = [
|
||||
'-c TimeZone=UTC',
|
||||
'-c DateStyle=ISO',
|
||||
'-c IntervalStyle=postgres',
|
||||
'-c extra_float_digits=3',
|
||||
'-c bytea_output=hex',
|
||||
].join(' ');
|
||||
|
||||
let srcPool = null;
|
||||
let dstPool = null;
|
||||
|
||||
function makePool(prefix, max) {
|
||||
const required = ['HOST', 'DB', 'USER', 'PASS'];
|
||||
const missing = required.filter(k => !process.env[`${prefix}_${k}`]);
|
||||
if (missing.length) throw new Error(`Variables manquantes : ${missing.map(k => `${prefix}_${k}`).join(', ')}`);
|
||||
const pool = new Pool({
|
||||
host: process.env[`${prefix}_HOST`],
|
||||
port: parseInt(process.env[`${prefix}_PORT`], 10) || 5432,
|
||||
database: process.env[`${prefix}_DB`],
|
||||
user: process.env[`${prefix}_USER`],
|
||||
password: process.env[`${prefix}_PASS`],
|
||||
max,
|
||||
connectionTimeoutMillis: 15000,
|
||||
idleTimeoutMillis: 60000,
|
||||
keepAlive: true,
|
||||
application_name: 'ff_sync_hostinger',
|
||||
options: SESSION_OPTIONS,
|
||||
});
|
||||
// Une connexion inactive qui tombe (tunnel frp coupé…) ne doit pas tuer le process
|
||||
pool.on('error', err => console.error(`[pool ${prefix}] ${err.message}`));
|
||||
return pool;
|
||||
}
|
||||
|
||||
/** Source : PostgreSQL Debian via le tunnel frps:15432 (lecture seule). */
|
||||
function getSrc() {
|
||||
if (!srcPool) {
|
||||
srcPool = new Pool({
|
||||
host: process.env.SRC_HOST,
|
||||
port: parseInt(process.env.SRC_PORT) || 5432,
|
||||
database: process.env.SRC_DB,
|
||||
user: process.env.SRC_USER,
|
||||
password: process.env.SRC_PASS,
|
||||
max: 3,
|
||||
connectionTimeoutMillis: 10000,
|
||||
});
|
||||
}
|
||||
if (!srcPool) srcPool = makePool('SRC', 3);
|
||||
return srcPool;
|
||||
}
|
||||
|
||||
/** Destination : PostgreSQL Hostinger (container postgres_db). */
|
||||
function getDst() {
|
||||
if (!dstPool) {
|
||||
dstPool = new Pool({
|
||||
host: process.env.DST_HOST,
|
||||
port: parseInt(process.env.DST_PORT) || 5432,
|
||||
database: process.env.DST_DB,
|
||||
user: process.env.DST_USER,
|
||||
password: process.env.DST_PASS,
|
||||
max: 5,
|
||||
});
|
||||
}
|
||||
if (!dstPool) dstPool = makePool('DST', 6);
|
||||
return dstPool;
|
||||
}
|
||||
|
||||
async function closeAll() {
|
||||
if (srcPool) { await srcPool.end(); srcPool = null; }
|
||||
if (dstPool) { await dstPool.end(); dstPool = null; }
|
||||
const p = [];
|
||||
if (srcPool) { p.push(srcPool.end()); srcPool = null; }
|
||||
if (dstPool) { p.push(dstPool.end()); dstPool = null; }
|
||||
await Promise.allSettled(p);
|
||||
}
|
||||
|
||||
module.exports = { getSrc, getDst, closeAll };
|
||||
module.exports = { getSrc, getDst, closeAll, SESSION_OPTIONS };
|
||||
+51
-76
@@ -1,84 +1,59 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Point d'entrée du service ff_sync_hostinger.
|
||||
*
|
||||
* node index.js → dashboard + cron (SYNC_CRON, fuseau SYNC_TZ)
|
||||
* node index.js --once → une exécution normale puis sortie
|
||||
* node index.js --force → une exécution forcée (garde-fou volume ignoré,
|
||||
* tables "fenêtre" rechargées entièrement) puis sortie
|
||||
*
|
||||
* Le verrou consultatif PostgreSQL empêche toute exécution concurrente,
|
||||
* y compris entre le service et une commande `docker exec … node index.js --once`.
|
||||
*/
|
||||
const cron = require('node-cron');
|
||||
const { getSrc, getDst, closeAll } = require('./db');
|
||||
const { batchUpsert, fullRefresh, logSync } = require('./utils');
|
||||
const { TABLES } = require('./tables');
|
||||
const {
|
||||
startDashboard, registerSyncFn,
|
||||
setSyncRunning, isSyncRunning,
|
||||
setProgress, clearProgress,
|
||||
} = require('./server');
|
||||
const { config } = require('./config');
|
||||
const log = require('./logger');
|
||||
const { closeAll } = require('./db');
|
||||
const { runSync } = require('./sync');
|
||||
const { startDashboard, registerSyncFn, registerNextRunFn } = require('./server');
|
||||
|
||||
const forceMode = process.argv.includes('--force');
|
||||
const args = process.argv.slice(2);
|
||||
const force = args.includes('--force');
|
||||
const once = force || args.includes('--once');
|
||||
|
||||
async function syncTable(src, dst, table, pk) {
|
||||
const { rows } = await src.query(`SELECT * FROM ${table}`);
|
||||
if (!rows.length) {
|
||||
await logSync(dst, table, 0, 'ok');
|
||||
console.log(`[${table}] 0 lignes (table vide)`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
let count;
|
||||
if (pk.length === 0) {
|
||||
count = await fullRefresh(dst, table, rows);
|
||||
} else {
|
||||
count = await batchUpsert(dst, table, rows, pk);
|
||||
}
|
||||
await logSync(dst, table, count, 'ok');
|
||||
console.log(`[${table}] ${count} lignes`);
|
||||
return count;
|
||||
}
|
||||
|
||||
async function syncAll(force = false) {
|
||||
if (isSyncRunning()) {
|
||||
console.log('Sync déjà en cours — ignorée.');
|
||||
return;
|
||||
}
|
||||
setSyncRunning(true);
|
||||
clearProgress();
|
||||
const start = Date.now();
|
||||
console.log(`\n=== SYNC START ${new Date().toISOString()} ===`);
|
||||
|
||||
const src = getSrc();
|
||||
const dst = getDst();
|
||||
|
||||
try {
|
||||
for (let i = 0; i < TABLES.length; i++) {
|
||||
const { name, pk } = TABLES[i];
|
||||
setProgress(name, i + 1);
|
||||
console.log(`[${i + 1}/${TABLES.length}] Sync ${name}…`);
|
||||
try {
|
||||
await syncTable(src, dst, name, pk);
|
||||
} catch (err) {
|
||||
await logSync(dst, name, 0, 'error', err.message).catch(() => {});
|
||||
console.error(`[${name}] ERREUR: ${err.message}`);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
setSyncRunning(false);
|
||||
clearProgress();
|
||||
}
|
||||
|
||||
const elapsed = ((Date.now() - start) / 1000).toFixed(1);
|
||||
console.log(`=== SYNC DONE en ${elapsed}s ===\n`);
|
||||
}
|
||||
|
||||
registerSyncFn(syncAll);
|
||||
|
||||
if (forceMode) {
|
||||
console.log('Mode force : sync complète en cours...');
|
||||
syncAll(true)
|
||||
.then(() => closeAll())
|
||||
.then(() => process.exit(0))
|
||||
.catch(err => {
|
||||
console.error('Erreur fatale sync:', err.message);
|
||||
if (once) {
|
||||
runSync({ force, trigger: 'cli' })
|
||||
.then(async r => {
|
||||
await closeAll();
|
||||
process.exit(r.skipped || r.status === 'ok' ? 0 : 1);
|
||||
})
|
||||
.catch(async err => {
|
||||
log.error(`Erreur fatale : ${err.message}`);
|
||||
await closeAll();
|
||||
process.exit(1);
|
||||
});
|
||||
} else {
|
||||
startDashboard();
|
||||
console.log('Service sync Hostinger démarré — cron: 0 7 * * * (7h00 chaque nuit)');
|
||||
cron.schedule('0 7 * * *', () => {
|
||||
syncAll(false).catch(err => console.error('Erreur cron sync:', err.message));
|
||||
});
|
||||
if (!cron.validate(config.cron)) {
|
||||
log.error(`SYNC_CRON invalide : "${config.cron}"`);
|
||||
process.exit(1);
|
||||
}
|
||||
registerSyncFn(runSync);
|
||||
const task = cron.schedule(
|
||||
config.cron,
|
||||
() => runSync({ force: false, trigger: 'cron' }).catch(err => log.error(`Sync cron : ${err.message}`)),
|
||||
{ timezone: config.timeZone, name: 'ff-sync', noOverlap: true },
|
||||
);
|
||||
registerNextRunFn(() => task.getNextRun());
|
||||
const server = startDashboard();
|
||||
log.info(`Service sync Hostinger démarré — cron "${config.cron}" (${config.timeZone}), prochaine exécution ${task.getNextRun()?.toISOString() ?? '?'}`);
|
||||
|
||||
const shutdown = sig => {
|
||||
log.info(`${sig} reçu — arrêt (une transaction en cours est annulée par PostgreSQL)`);
|
||||
task.stop();
|
||||
server.close();
|
||||
closeAll().finally(() => process.exit(0));
|
||||
setTimeout(() => process.exit(0), 5000).unref();
|
||||
};
|
||||
process.on('SIGTERM', () => shutdown('SIGTERM'));
|
||||
process.on('SIGINT', () => shutdown('SIGINT'));
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Journal applicatif : sortie console + tampon circulaire affiché par le dashboard.
|
||||
* Les messages sont du texte brut ; le dashboard les insère via textContent (jamais innerHTML).
|
||||
*/
|
||||
const MAX_LINES = 500;
|
||||
const MAX_LEN = 4000;
|
||||
const buffer = [];
|
||||
|
||||
function push(level, args) {
|
||||
const msg = args
|
||||
.map(a => (a instanceof Error ? a.message : typeof a === 'string' ? a : JSON.stringify(a)))
|
||||
.join(' ')
|
||||
.slice(0, MAX_LEN);
|
||||
buffer.push({ ts: new Date().toISOString(), level, msg });
|
||||
if (buffer.length > MAX_LINES) buffer.shift();
|
||||
const out = level === 'error' ? console.error : level === 'warn' ? console.warn : console.log;
|
||||
out(`[${level}] ${msg}`);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
info: (...a) => push('info', a),
|
||||
warn: (...a) => push('warn', a),
|
||||
error: (...a) => push('error', a),
|
||||
getLogs: (limit = 200) => buffer.slice(-Math.min(Math.max(1, limit | 0), MAX_LINES)),
|
||||
};
|
||||
Generated
+1015
File diff suppressed because it is too large.
Load diff
+14
-4
@@ -1,13 +1,23 @@
|
||||
{
|
||||
"name": "ff-sync-hostinger",
|
||||
"version": "1.0.0",
|
||||
"description": "Sync nightly PG Debian → PG Hostinger",
|
||||
"version": "2.0.0",
|
||||
"private": true,
|
||||
"description": "Miroir PG Debian → PG Hostinger (schéma automatique, copie transactionnelle/incrémentale)",
|
||||
"main": "index.js",
|
||||
"license": "ISC",
|
||||
"type": "commonjs",
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "node index.js",
|
||||
"sync": "node index.js --once",
|
||||
"sync:force": "node index.js --force",
|
||||
"test": "node --test test/*.test.js"
|
||||
},
|
||||
"dependencies": {
|
||||
"express": "^5.1.0",
|
||||
"node-cron": "^3.0.3",
|
||||
"pg": "^8.13.3"
|
||||
"node-cron": "^4.6.0",
|
||||
"pg": "^8.16.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
'use strict';
|
||||
// Dashboard sync Hostinger — toutes les données dynamiques passent par textContent (pas d'innerHTML).
|
||||
|
||||
const $ = id => document.getElementById(id);
|
||||
|
||||
function el(tag, opts = {}, children = []) {
|
||||
const e = document.createElement(tag);
|
||||
if (opts.className) e.className = opts.className;
|
||||
if (opts.text != null) e.textContent = String(opts.text);
|
||||
if (opts.title) e.title = String(opts.title);
|
||||
if (opts.colSpan) e.colSpan = opts.colSpan;
|
||||
for (const c of children) e.appendChild(c);
|
||||
return e;
|
||||
}
|
||||
|
||||
function fmtDate(iso) {
|
||||
if (!iso) return '—';
|
||||
const d = new Date(iso);
|
||||
return isNaN(d) ? '—' : d.toLocaleString('fr-FR');
|
||||
}
|
||||
function fmtDuration(ms) {
|
||||
if (ms == null) return '—';
|
||||
const s = Math.round(ms / 1000);
|
||||
return s < 60 ? s + ' s' : Math.floor(s / 60) + ' min ' + (s % 60) + ' s';
|
||||
}
|
||||
function fmtInt(n) {
|
||||
return n == null ? '—' : Number(n).toLocaleString('fr-FR');
|
||||
}
|
||||
|
||||
function renderProgress(data) {
|
||||
const box = $('progressBox');
|
||||
if (!data.running) { box.classList.add('hidden'); return; }
|
||||
box.classList.remove('hidden');
|
||||
const p = data.progress;
|
||||
const pct = p ? Math.round((p.index / p.total) * 100) : 0;
|
||||
$('progressBar').style.width = pct + '%';
|
||||
$('progressPct').textContent = pct + '%';
|
||||
$('progressLabel').textContent = 'Sync ' + (data.force ? 'FORCÉE ' : '') + 'en cours — ' +
|
||||
(data.phase || '') + (p ? ' — table ' + p.index + '/' + p.total : '');
|
||||
$('progressDetail').textContent = p
|
||||
? 'En cours : ' + p.current + ' | Durée : ' + fmtDuration(Date.now() - new Date(p.startedAt))
|
||||
: '';
|
||||
}
|
||||
|
||||
function renderRunInfo(data) {
|
||||
const run = data.tables.find(t => t.table_name === '_run');
|
||||
const box = $('runInfo');
|
||||
box.textContent = '';
|
||||
if (data.dbError) {
|
||||
box.appendChild(el('span', { className: 'st-error', text: 'Erreur : ' + data.dbError }));
|
||||
return;
|
||||
}
|
||||
if (!run) { box.textContent = 'Aucune exécution enregistrée.'; return; }
|
||||
box.appendChild(el('span', { text: 'Dernière exécution : ' }));
|
||||
box.appendChild(el('strong', { className: 'st-' + run.status, text: run.status }));
|
||||
box.appendChild(el('span', { text: ' — ' + fmtDate(run.last_sync) + ' — ' + fmtDuration(run.duration_ms) + ' — ' + (run.detail || '') }));
|
||||
if (run.error_msg) box.appendChild(el('div', { className: 'st-error', text: 'Tables en erreur : ' + run.error_msg }));
|
||||
}
|
||||
|
||||
function renderTables(data) {
|
||||
const body = $('tableBody');
|
||||
body.textContent = '';
|
||||
const rows = data.tables.filter(t => t.table_name !== '_run');
|
||||
if (!rows.length) {
|
||||
body.appendChild(el('tr', {}, [el('td', { colSpan: 7, text: 'Aucune table synchronisée pour le moment.' })]));
|
||||
return;
|
||||
}
|
||||
for (const t of rows) {
|
||||
const status = t.status || '—';
|
||||
body.appendChild(el('tr', {}, [
|
||||
el('td', {}, [el('strong', { text: t.table_name })]),
|
||||
el('td', {}, [el('span', { className: 'strat ' + (t.strategy || ''), text: t.strategy || '—' })]),
|
||||
el('td', { text: fmtDate(t.last_sync) }),
|
||||
el('td', { className: 'num', text: fmtInt(t.rows_synced) }),
|
||||
el('td', { className: 'num', text: fmtDuration(t.duration_ms) }),
|
||||
el('td', { className: 'st-' + status, text: status }),
|
||||
el('td', { className: 'detail', text: t.error_msg || t.detail || '' }),
|
||||
]));
|
||||
}
|
||||
}
|
||||
|
||||
function renderStatus(data) {
|
||||
const badge = $('statusBadge');
|
||||
badge.textContent = data.running ? 'En cours…' : 'Inactif';
|
||||
badge.className = 'badge ' + (data.running ? 'running' : 'idle');
|
||||
$('btnSync').disabled = data.running;
|
||||
$('btnForce').disabled = data.running;
|
||||
renderProgress(data);
|
||||
renderRunInfo(data);
|
||||
renderTables(data);
|
||||
}
|
||||
|
||||
function renderLogs(logs) {
|
||||
const box = $('logBox');
|
||||
const atBottom = box.scrollTop + box.clientHeight >= box.scrollHeight - 10;
|
||||
box.textContent = '';
|
||||
if (!logs.length) { box.textContent = 'Aucun log.'; return; }
|
||||
for (const l of logs) {
|
||||
const line = el('div', { className: 'log-line ' + (['info', 'warn', 'error'].includes(l.level) ? l.level : '') });
|
||||
line.appendChild(el('span', { className: 'ts', text: new Date(l.ts).toLocaleTimeString('fr-FR') }));
|
||||
line.appendChild(document.createTextNode(l.msg));
|
||||
box.appendChild(line);
|
||||
}
|
||||
if (atBottom) box.scrollTop = box.scrollHeight;
|
||||
}
|
||||
|
||||
async function getJson(url) {
|
||||
const r = await fetch(url, { cache: 'no-store' });
|
||||
if (!r.ok) throw new Error(url + ' : HTTP ' + r.status);
|
||||
return r.json();
|
||||
}
|
||||
|
||||
let lastStatus = null;
|
||||
async function refresh() {
|
||||
try {
|
||||
const [s, logs, next] = await Promise.all([getJson('/api/status'), getJson('/api/logs?limit=200'), getJson('/api/next')]);
|
||||
lastStatus = s;
|
||||
renderStatus(s);
|
||||
renderLogs(logs);
|
||||
$('nextSync').textContent = 'Prochaine sync auto : ' + fmtDate(next.next) + ' (' + next.cron + ', ' + next.timeZone + ')';
|
||||
$('lastRefresh').textContent = 'Mis à jour ' + new Date().toLocaleTimeString('fr-FR');
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
}
|
||||
}
|
||||
|
||||
function showMsg(ok, text) {
|
||||
const msg = $('msg');
|
||||
msg.textContent = text;
|
||||
msg.className = ok ? 'ok' : 'err';
|
||||
setTimeout(() => msg.classList.add('hidden'), 6000);
|
||||
}
|
||||
|
||||
async function runSync(force) {
|
||||
if (force && !confirm('Forcer la sync ?\n\nLe garde-fou de volume sera ignoré et les tables de mouvements (mvtart, mvtreg…) seront entièrement rechargées. Cela peut prendre longtemps.')) return;
|
||||
try {
|
||||
const r = await fetch('/api/sync/run', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json', 'X-Requested-By': 'ff-dashboard' },
|
||||
body: JSON.stringify({ force: !!force }),
|
||||
});
|
||||
const data = await r.json().catch(() => ({}));
|
||||
showMsg(r.ok, r.ok ? data.message : (data.error || 'HTTP ' + r.status));
|
||||
} catch (e) {
|
||||
showMsg(false, e.message);
|
||||
}
|
||||
setTimeout(refresh, 800);
|
||||
}
|
||||
|
||||
function copyLogs() {
|
||||
const text = Array.from($('logBox').querySelectorAll('.log-line')).map(l => l.textContent).join('\n');
|
||||
const btn = $('btnCopyLogs');
|
||||
const done = () => { btn.textContent = 'Copié !'; setTimeout(() => { btn.textContent = 'Copier les logs'; }, 2000); };
|
||||
if (navigator.clipboard && window.isSecureContext) {
|
||||
navigator.clipboard.writeText(text).then(done).catch(() => alert('Copie impossible : sélectionnez les logs puis Ctrl+C'));
|
||||
} else {
|
||||
alert('Copie impossible hors HTTPS : sélectionnez les logs puis Ctrl+C');
|
||||
}
|
||||
}
|
||||
|
||||
$('btnSync').addEventListener('click', () => runSync(false));
|
||||
$('btnForce').addEventListener('click', () => runSync(true));
|
||||
$('btnRefresh').addEventListener('click', refresh);
|
||||
$('btnCopyLogs').addEventListener('click', copyLogs);
|
||||
|
||||
async function loop() {
|
||||
await refresh();
|
||||
setTimeout(loop, lastStatus && lastStatus.running ? 3000 : 15000);
|
||||
}
|
||||
loop();
|
||||
@@ -0,0 +1,102 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Sync Hostinger</title>
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:system-ui,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh}
|
||||
header{background:#1e293b;padding:1rem 2rem;display:flex;align-items:center;gap:1rem;flex-wrap:wrap;border-bottom:1px solid #334155}
|
||||
header h1{font-size:1.2rem;font-weight:700;color:#f8fafc}
|
||||
.subtitle{font-size:.75rem;color:#64748b}
|
||||
.badge{font-size:.75rem;padding:.25rem .6rem;border-radius:9999px;font-weight:600}
|
||||
.badge.idle{background:#1e3a5f;color:#93c5fd}
|
||||
.badge.running{background:#713f12;color:#fde68a;animation:pulse 1s infinite}
|
||||
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.5}}
|
||||
main{padding:1.5rem 2rem;max-width:1300px;margin:0 auto}
|
||||
.actions{display:flex;align-items:center;gap:1rem;flex-wrap:wrap;margin-bottom:1.5rem}
|
||||
button{padding:.55rem 1.3rem;border:none;border-radius:.5rem;cursor:pointer;font-weight:600;font-size:.85rem;transition:.15s}
|
||||
button:disabled{opacity:.4;cursor:not-allowed}
|
||||
#btnSync{background:#2563eb;color:#fff}
|
||||
#btnSync:hover:not(:disabled){background:#1d4ed8}
|
||||
#btnForce{background:#7c2d12;color:#fed7aa}
|
||||
#btnForce:hover:not(:disabled){background:#9a3412}
|
||||
#btnRefresh,#btnCopyLogs{background:#334155;color:#cbd5e1}
|
||||
#btnRefresh:hover,#btnCopyLogs:hover{background:#475569}
|
||||
#btnCopyLogs{font-size:.75rem;padding:.35rem .8rem}
|
||||
#nextSync{font-size:.8rem;color:#64748b;margin-left:auto}
|
||||
#msg{font-size:.82rem;padding:.35rem .75rem;border-radius:.4rem}
|
||||
#msg.ok{background:#14532d;color:#86efac}
|
||||
#msg.err{background:#450a0a;color:#fca5a5}
|
||||
.hidden{display:none !important}
|
||||
.box{background:#1e293b;border:1px solid #334155;border-radius:.75rem;padding:1rem 1.25rem;margin-bottom:1.5rem}
|
||||
.progress-header{display:flex;justify-content:space-between;margin-bottom:.6rem;font-size:.85rem}
|
||||
.progress-bar-bg{background:#0f172a;border-radius:9999px;height:12px;overflow:hidden}
|
||||
.progress-bar-fill{height:100%;background:linear-gradient(90deg,#059669,#2563eb);border-radius:9999px;transition:width .5s ease;width:0}
|
||||
.progress-detail{margin-top:.5rem;font-size:.78rem;color:#64748b}
|
||||
#runInfo{font-size:.85rem}
|
||||
.card{background:#1e293b;border:1px solid #334155;border-radius:.75rem;overflow:hidden;margin-bottom:1.5rem}
|
||||
.card-header{padding:.65rem 1rem;background:#0f172a;border-bottom:1px solid #334155;font-size:.75rem;font-weight:600;text-transform:uppercase;letter-spacing:.05em;color:#94a3b8;display:flex;justify-content:space-between;align-items:center}
|
||||
.table-wrap{overflow-x:auto}
|
||||
table{width:100%;border-collapse:collapse;font-size:.82rem}
|
||||
th{padding:.5rem .8rem;text-align:left;color:#64748b;font-weight:500;border-bottom:1px solid #334155;white-space:nowrap}
|
||||
td{padding:.5rem .8rem;border-bottom:1px solid #263449;vertical-align:top}
|
||||
td.num{text-align:right;font-variant-numeric:tabular-nums;white-space:nowrap}
|
||||
td.detail{color:#94a3b8;max-width:520px;word-break:break-word}
|
||||
.st-ok{color:#4ade80}.st-error{color:#f87171}.st-running{color:#fde68a}
|
||||
.strat{font-size:.72rem;padding:.1rem .45rem;border-radius:.3rem;background:#334155;color:#cbd5e1}
|
||||
.strat.window{background:#1e3a5f;color:#93c5fd}
|
||||
.log-box{height:340px;overflow-y:auto;font-family:monospace;font-size:.76rem;padding:.75rem 1rem;background:#0a0f1e;color:#a5f3fc;line-height:1.7}
|
||||
.log-line{white-space:pre-wrap;word-break:break-all}
|
||||
.log-line.error{color:#f87171}
|
||||
.log-line.warn{color:#fbbf24}
|
||||
.ts{color:#475569;margin-right:.5rem}
|
||||
#lastRefresh{font-size:.75rem;color:#475569;margin-left:auto}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Sync Hostinger</h1>
|
||||
<span class="subtitle">PG Debian (frps:15432) → PG Hostinger (miroir automatique du schéma)</span>
|
||||
<span id="statusBadge" class="badge idle">Inactif</span>
|
||||
<span id="lastRefresh"></span>
|
||||
</header>
|
||||
<main>
|
||||
<div class="actions">
|
||||
<button id="btnSync" type="button">Lancer la sync</button>
|
||||
<button id="btnForce" type="button" title="Ignore le garde-fou volume et recharge entièrement les tables de mouvements">Forcer (rechargement complet)</button>
|
||||
<button id="btnRefresh" type="button">Rafraîchir</button>
|
||||
<span id="msg" class="hidden"></span>
|
||||
<span id="nextSync"></span>
|
||||
</div>
|
||||
|
||||
<div id="progressBox" class="box hidden">
|
||||
<div class="progress-header">
|
||||
<span id="progressLabel">Synchronisation en cours…</span>
|
||||
<span id="progressPct">0%</span>
|
||||
</div>
|
||||
<div class="progress-bar-bg"><div class="progress-bar-fill" id="progressBar"></div></div>
|
||||
<div class="progress-detail" id="progressDetail"></div>
|
||||
</div>
|
||||
|
||||
<div class="box"><div id="runInfo">Chargement…</div></div>
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header"><span>Statut des tables</span></div>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead><tr><th>Table</th><th>Stratégie</th><th>Dernière sync</th><th>Lignes</th><th>Durée</th><th>Statut</th><th>Détail / erreur</th></tr></thead>
|
||||
<tbody id="tableBody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card">
|
||||
<div class="card-header"><span>Logs</span><button id="btnCopyLogs" type="button">Copier les logs</button></div>
|
||||
<div class="log-box" id="logBox">Chargement…</div>
|
||||
</div>
|
||||
</main>
|
||||
<script src="/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Miroir automatique du schéma source → destination.
|
||||
* Aucune liste de tables ni schéma écrit à la main : tout est lu dans pg_catalog
|
||||
* à chaque exécution. On ne supprime jamais rien côté destination.
|
||||
*/
|
||||
const log = require('./logger');
|
||||
const {
|
||||
qt, buildCreateTable, buildAddColumn, buildAddPrimaryKey,
|
||||
indexDefIfNotExists, needsTrgm,
|
||||
} = require('./sql');
|
||||
|
||||
const COLUMNS_SQL = `
|
||||
SELECT c.relname AS table_name, a.attname AS name,
|
||||
pg_catalog.format_type(a.atttypid, a.atttypmod) AS type,
|
||||
a.attnotnull AS notnull
|
||||
FROM pg_catalog.pg_attribute a
|
||||
JOIN pg_catalog.pg_class c ON c.oid = a.attrelid
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = 'public'
|
||||
AND c.relkind IN ('r', 'p')
|
||||
AND NOT c.relispartition
|
||||
AND a.attnum > 0
|
||||
AND NOT a.attisdropped
|
||||
ORDER BY c.relname, a.attnum`;
|
||||
|
||||
const PK_SQL = `
|
||||
SELECT c.relname AS table_name,
|
||||
array_agg(a.attname::text ORDER BY k.ord) AS pk
|
||||
FROM pg_catalog.pg_constraint con
|
||||
JOIN pg_catalog.pg_class c ON c.oid = con.conrelid
|
||||
JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
|
||||
CROSS JOIN LATERAL unnest(con.conkey) WITH ORDINALITY AS k(attnum, ord)
|
||||
JOIN pg_catalog.pg_attribute a ON a.attrelid = c.oid AND a.attnum = k.attnum
|
||||
WHERE n.nspname = 'public' AND con.contype = 'p'
|
||||
GROUP BY c.relname`;
|
||||
|
||||
// Index hors clé primaire (les index des contraintes UNIQUE deviennent des index uniques)
|
||||
const INDEXES_SQL = `
|
||||
SELECT i.tablename AS table_name, i.indexname AS name, i.indexdef AS def
|
||||
FROM pg_catalog.pg_indexes i
|
||||
JOIN pg_catalog.pg_namespace n ON n.nspname = i.schemaname
|
||||
JOIN pg_catalog.pg_class ic ON ic.relname = i.indexname AND ic.relnamespace = n.oid
|
||||
JOIN pg_catalog.pg_index x ON x.indexrelid = ic.oid
|
||||
WHERE i.schemaname = 'public' AND NOT x.indisprimary
|
||||
ORDER BY i.tablename, i.indexname`;
|
||||
|
||||
/**
|
||||
* Lit le catalogue d'une base : Map table → { name, columns:[{name,type,notnull}], pk:[], indexes:[{name,def}] }.
|
||||
*/
|
||||
async function readCatalog(client) {
|
||||
const tables = new Map();
|
||||
const { rows: cols } = await client.query(COLUMNS_SQL);
|
||||
for (const r of cols) {
|
||||
if (!tables.has(r.table_name)) tables.set(r.table_name, { name: r.table_name, columns: [], pk: [], indexes: [] });
|
||||
tables.get(r.table_name).columns.push({ name: r.name, type: r.type, notnull: r.notnull });
|
||||
}
|
||||
const { rows: pks } = await client.query(PK_SQL);
|
||||
for (const r of pks) if (tables.has(r.table_name)) tables.get(r.table_name).pk = r.pk;
|
||||
const { rows: idx } = await client.query(INDEXES_SQL);
|
||||
for (const r of idx) if (tables.has(r.table_name)) tables.get(r.table_name).indexes.push({ name: r.name, def: r.def });
|
||||
return tables;
|
||||
}
|
||||
|
||||
/**
|
||||
* Crée la table destination si absente, ajoute les colonnes manquantes,
|
||||
* signale (sans modifier) les colonnes de type différent.
|
||||
* Retourne la liste des avertissements.
|
||||
*/
|
||||
async function mirrorTable(dst, srcTable, dstName, dstCatalog) {
|
||||
const warnings = [];
|
||||
const existing = dstCatalog.get(dstName);
|
||||
if (!existing) {
|
||||
await dst.query(buildCreateTable(dstName, srcTable.columns, srcTable.pk));
|
||||
log.info(`[${dstName}] table créée (${srcTable.columns.length} colonnes)`);
|
||||
dstCatalog.set(dstName, {
|
||||
name: dstName,
|
||||
columns: srcTable.columns.map(c => ({ ...c })),
|
||||
pk: [...srcTable.pk],
|
||||
indexes: [],
|
||||
});
|
||||
return warnings;
|
||||
}
|
||||
const dstCols = new Map(existing.columns.map(c => [c.name, c]));
|
||||
for (const col of srcTable.columns) {
|
||||
const d = dstCols.get(col.name);
|
||||
if (!d) {
|
||||
await dst.query(buildAddColumn(dstName, col));
|
||||
existing.columns.push({ name: col.name, type: col.type, notnull: false });
|
||||
log.info(`[${dstName}] colonne ajoutée : ${col.name} ${col.type}`);
|
||||
} else if (d.type !== col.type) {
|
||||
const w = `type différent pour ${col.name} : source ${col.type} / destination ${d.type} (non modifié)`;
|
||||
warnings.push(w);
|
||||
log.warn(`[${dstName}] ${w}`);
|
||||
}
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
/** Tente CREATE EXTENSION pg_trgm si un index source en a besoin. Retourne true si disponible. */
|
||||
async function ensureTrgm(dst, srcCatalog) {
|
||||
const needed = [...srcCatalog.values()].some(t => t.indexes.some(i => needsTrgm(i.def)));
|
||||
if (!needed) return true;
|
||||
try {
|
||||
await dst.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
|
||||
return true;
|
||||
} catch (err) {
|
||||
log.warn(`pg_trgm indisponible sur la destination (${err.message}) — index trigrammes ignorés`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Après la copie des données : crée les index manquants (même nom, IF NOT EXISTS)
|
||||
* et la clé primaire si la destination n'en a pas. Les échecs sont des avertissements.
|
||||
* Les index ne sont reproduits que si le nom de table est identique (pas pour source_sync_log).
|
||||
*/
|
||||
async function mirrorIndexes(dst, srcTable, dstName, dstCatalog, trgmOk) {
|
||||
const warnings = [];
|
||||
const existing = dstCatalog.get(dstName);
|
||||
if (srcTable.pk.length && existing && !existing.pk.length) {
|
||||
try {
|
||||
await dst.query(buildAddPrimaryKey(dstName, srcTable.pk));
|
||||
existing.pk = [...srcTable.pk];
|
||||
log.info(`[${dstName}] clé primaire ajoutée (${srcTable.pk.join(', ')})`);
|
||||
} catch (err) {
|
||||
const w = `clé primaire (${srcTable.pk.join(', ')}) non ajoutée : ${err.message}`;
|
||||
warnings.push(w);
|
||||
log.warn(`[${dstName}] ${w}`);
|
||||
}
|
||||
}
|
||||
if (dstName !== srcTable.name) return warnings;
|
||||
const have = new Set((existing?.indexes || []).map(i => i.name));
|
||||
for (const idx of srcTable.indexes) {
|
||||
if (have.has(idx.name)) continue;
|
||||
if (!trgmOk && needsTrgm(idx.def)) {
|
||||
warnings.push(`index ${idx.name} ignoré (pg_trgm indisponible)`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await dst.query(indexDefIfNotExists(idx.def));
|
||||
existing?.indexes.push({ name: idx.name, def: idx.def });
|
||||
log.info(`[${dstName}] index ${idx.name} OK`);
|
||||
} catch (err) {
|
||||
const w = `index ${idx.name} non créé : ${err.message}`;
|
||||
warnings.push(w);
|
||||
log.warn(`[${dstName}] ${w}`);
|
||||
}
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
/** Table de suivi côté destination (mêmes colonnes que la source + colonnes de détail). */
|
||||
async function ensureDstSyncLog(dst) {
|
||||
await dst.query(`CREATE TABLE IF NOT EXISTS ${qt('sync_log')} (
|
||||
table_name TEXT PRIMARY KEY,
|
||||
last_sync TIMESTAMP,
|
||||
rows_synced INTEGER,
|
||||
status TEXT,
|
||||
error_msg TEXT
|
||||
)`);
|
||||
for (const [name, type] of [['strategy', 'TEXT'], ['duration_ms', 'INTEGER'], ['detail', 'TEXT']]) {
|
||||
await dst.query(`ALTER TABLE ${qt('sync_log')} ADD COLUMN IF NOT EXISTS "${name}" ${type}`);
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { readCatalog, mirrorTable, ensureTrgm, mirrorIndexes, ensureDstSyncLog };
|
||||
+103
-248
@@ -1,281 +1,136 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Dashboard de la sync Hostinger (port 3002, joint par Nginx Proxy Manager sur nginx_default).
|
||||
*
|
||||
* Sécurité :
|
||||
* - HTTP Basic auth (DASHBOARD_USER / DASHBOARD_PASSWORD), comparaison à temps constant ;
|
||||
* sans ces variables, tout répond 503 sauf GET /healthz (healthcheck Docker).
|
||||
* - POST /api/sync/run exige l'en-tête X-Requested-By (bloque les formulaires cross-site / CSRF).
|
||||
* - Aucune donnée dynamique insérée via innerHTML ; CSP sans script inline.
|
||||
*/
|
||||
const crypto = require('node:crypto');
|
||||
const path = require('node:path');
|
||||
const express = require('express');
|
||||
const { config } = require('./config');
|
||||
const log = require('./logger');
|
||||
const { getDst } = require('./db');
|
||||
const { TABLES } = require('./tables');
|
||||
const { state } = require('./sync');
|
||||
|
||||
const app = express();
|
||||
const PORT = process.env.SYNC_DASHBOARD_PORT || 3002;
|
||||
const app = express();
|
||||
app.disable('x-powered-by');
|
||||
app.set('trust proxy', 'loopback, uniquelocal');
|
||||
|
||||
const TABLES_ORDER = TABLES.map(t => t.name);
|
||||
const TABLES_TOTAL = TABLES_ORDER.length;
|
||||
let runFn = null;
|
||||
let nextRunFn = () => null;
|
||||
function registerSyncFn(fn) { runFn = fn; }
|
||||
function registerNextRunFn(fn) { nextRunFn = fn; }
|
||||
|
||||
const state = {
|
||||
running: false,
|
||||
progress: null,
|
||||
};
|
||||
app.get('/healthz', (req, res) => res.json({ ok: true }));
|
||||
|
||||
const logBuffer = [];
|
||||
function pushLog(msg, level = 'info') {
|
||||
logBuffer.push({ ts: new Date().toISOString(), msg, level });
|
||||
if (logBuffer.length > 300) logBuffer.shift();
|
||||
const sha = s => crypto.createHash('sha256').update(String(s)).digest();
|
||||
function safeEqual(a, b) {
|
||||
return crypto.timingSafeEqual(sha(a), sha(b));
|
||||
}
|
||||
|
||||
const _log = console.log.bind(console);
|
||||
const _err = console.error.bind(console);
|
||||
console.log = (...a) => { pushLog(a.join(' '), 'info'); _log(...a); };
|
||||
console.error = (...a) => { pushLog(a.join(' '), 'error'); _err(...a); };
|
||||
|
||||
function setProgress(tableName, index) {
|
||||
state.progress = {
|
||||
current: tableName,
|
||||
index,
|
||||
total: TABLES_TOTAL,
|
||||
pct: Math.round((index / TABLES_TOTAL) * 100),
|
||||
startedAt: state.progress?.startedAt || new Date().toISOString(),
|
||||
};
|
||||
function basicAuth(req, res, next) {
|
||||
if (!config.dashboardUser || !config.dashboardPass) {
|
||||
return res.status(503).type('text/plain; charset=utf-8')
|
||||
.send('dashboard désactivé: définir DASHBOARD_USER/DASHBOARD_PASSWORD');
|
||||
}
|
||||
const header = req.get('authorization') || '';
|
||||
const m = /^Basic\s+([A-Za-z0-9+/=]+)$/i.exec(header);
|
||||
let user = '';
|
||||
let pass = '';
|
||||
if (m) {
|
||||
const decoded = Buffer.from(m[1], 'base64').toString('utf8');
|
||||
const i = decoded.indexOf(':');
|
||||
if (i >= 0) { user = decoded.slice(0, i); pass = decoded.slice(i + 1); }
|
||||
}
|
||||
// Les deux comparaisons sont toujours évaluées (pas de court-circuit)
|
||||
const okUser = safeEqual(user, config.dashboardUser);
|
||||
const okPass = safeEqual(pass, config.dashboardPass);
|
||||
if (m && okUser && okPass) return next();
|
||||
if (m) log.warn(`Authentification dashboard refusée (ip ${req.ip})`);
|
||||
res.set('WWW-Authenticate', 'Basic realm="ff_sync_hostinger", charset="UTF-8"');
|
||||
return res.status(401).type('text/plain; charset=utf-8').send('Authentification requise');
|
||||
}
|
||||
function clearProgress() { state.progress = null; }
|
||||
|
||||
let syncAllFn = null;
|
||||
function registerSyncFn(fn) { syncAllFn = fn; }
|
||||
function setSyncRunning(val) { state.running = val; }
|
||||
function isSyncRunning() { return state.running; }
|
||||
|
||||
app.use(express.json());
|
||||
app.use(basicAuth);
|
||||
app.use((req, res, next) => {
|
||||
res.set({
|
||||
'Content-Security-Policy': "default-src 'none'; script-src 'self'; style-src 'self' 'unsafe-inline'; connect-src 'self'; img-src 'self'; frame-ancestors 'none'; base-uri 'none'; form-action 'none'",
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'Referrer-Policy': 'no-referrer',
|
||||
'Cache-Control': 'no-store',
|
||||
});
|
||||
next();
|
||||
});
|
||||
app.use(express.json({ limit: '10kb' }));
|
||||
|
||||
app.get('/api/status', async (req, res) => {
|
||||
let tables = [];
|
||||
let dbError = null;
|
||||
try {
|
||||
const dst = getDst();
|
||||
const { rows } = await dst.query(
|
||||
`SELECT table_name, last_sync, rows_synced, status, error_msg
|
||||
FROM sync_log ORDER BY table_name`
|
||||
const { rows } = await getDst().query(
|
||||
`SELECT table_name, last_sync, rows_synced, status, error_msg, strategy, duration_ms, detail
|
||||
FROM public.sync_log ORDER BY table_name`,
|
||||
);
|
||||
res.json({ running: state.running, progress: state.progress, tables: rows });
|
||||
tables = rows;
|
||||
} catch (err) {
|
||||
res.status(500).json({ error: err.message });
|
||||
// 42P01 = sync_log pas encore créée (aucune exécution)
|
||||
if (err.code !== '42P01') dbError = 'base destination injoignable';
|
||||
if (err.code !== '42P01') log.error(`/api/status : ${err.message}`);
|
||||
}
|
||||
res.json({
|
||||
running: state.running,
|
||||
force: state.force,
|
||||
phase: state.phase,
|
||||
progress: state.progress,
|
||||
lastRun: state.lastRun,
|
||||
tables,
|
||||
dbError,
|
||||
});
|
||||
});
|
||||
|
||||
app.get('/api/logs', (req, res) => {
|
||||
const limit = parseInt(req.query.limit) || 150;
|
||||
res.json(logBuffer.slice(-limit));
|
||||
res.json(log.getLogs(parseInt(req.query.limit, 10) || 200));
|
||||
});
|
||||
|
||||
app.get('/api/next', (req, res) => {
|
||||
const now = new Date();
|
||||
const next = new Date(now);
|
||||
next.setHours(5, 30, 0, 0);
|
||||
if (next <= now) next.setDate(next.getDate() + 1);
|
||||
res.json({ next: next.toISOString() });
|
||||
const next = nextRunFn();
|
||||
res.json({ next: next ? next.toISOString() : null, cron: config.cron, timeZone: config.timeZone });
|
||||
});
|
||||
|
||||
app.post('/api/sync/run', async (req, res) => {
|
||||
app.post('/api/sync/run', (req, res) => {
|
||||
if (req.get('x-requested-by') !== 'ff-dashboard') {
|
||||
return res.status(400).json({ error: 'en-tête X-Requested-By manquant' });
|
||||
}
|
||||
if (state.running) return res.status(409).json({ error: 'Sync déjà en cours' });
|
||||
if (!syncAllFn) return res.status(500).json({ error: 'syncAll non enregistré' });
|
||||
res.json({ message: 'Sync lancée' });
|
||||
syncAllFn(true).catch(err => console.error('Erreur sync manuelle:', err.message));
|
||||
if (!runFn) return res.status(500).json({ error: 'sync non initialisée' });
|
||||
const force = req.body?.force === true;
|
||||
log.info(`Sync ${force ? 'FORCÉE ' : ''}demandée depuis le dashboard (ip ${req.ip})`);
|
||||
res.status(202).json({ message: force ? 'Sync forcée lancée' : 'Sync lancée' });
|
||||
runFn({ force, trigger: 'dashboard' }).catch(err => log.error(`Sync manuelle : ${err.message}`));
|
||||
});
|
||||
|
||||
app.get('/', (req, res) => res.send(HTML));
|
||||
const PUBLIC_DIR = path.join(__dirname, 'public');
|
||||
app.get('/', (req, res) => res.sendFile(path.join(PUBLIC_DIR, 'index.html')));
|
||||
app.get('/app.js', (req, res) => res.type('application/javascript').sendFile(path.join(PUBLIC_DIR, 'app.js')));
|
||||
|
||||
app.use((req, res) => res.status(404).json({ error: 'not found' }));
|
||||
// eslint-disable-next-line no-unused-vars
|
||||
app.use((err, req, res, next) => {
|
||||
log.error(`HTTP ${req.method} ${req.path} : ${err.message}`);
|
||||
res.status(500).json({ error: 'erreur interne' });
|
||||
});
|
||||
|
||||
function startDashboard() {
|
||||
app.listen(PORT, '0.0.0.0', () => {
|
||||
console.log(`Dashboard sync Hostinger : http://0.0.0.0:${PORT}`);
|
||||
if (!config.dashboardUser || !config.dashboardPass) {
|
||||
log.warn('DASHBOARD_USER / DASHBOARD_PASSWORD non définis : dashboard et API désactivés (503)');
|
||||
}
|
||||
return app.listen(config.port, '0.0.0.0', () => {
|
||||
log.info(`Dashboard sync Hostinger : port ${config.port}`);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { startDashboard, registerSyncFn, setSyncRunning, isSyncRunning, setProgress, clearProgress };
|
||||
|
||||
const HTML = `<!DOCTYPE html>
|
||||
<html lang="fr">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Sync Hostinger</title>
|
||||
<style>
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{font-family:system-ui,sans-serif;background:#0f172a;color:#e2e8f0;min-height:100vh}
|
||||
header{background:#1e293b;padding:1rem 2rem;display:flex;align-items:center;gap:1rem;border-bottom:1px solid #334155}
|
||||
header h1{font-size:1.2rem;font-weight:700;color:#f8fafc}
|
||||
.subtitle{font-size:.75rem;color:#64748b}
|
||||
.badge{font-size:.75rem;padding:.25rem .6rem;border-radius:9999px;font-weight:600}
|
||||
.badge.idle{background:#1e3a5f;color:#93c5fd}
|
||||
.badge.running{background:#713f12;color:#fde68a;animation:pulse 1s infinite}
|
||||
@keyframes pulse{0%,100%{opacity:1}50%{opacity:.5}}
|
||||
main{padding:1.5rem 2rem;max-width:1100px;margin:0 auto}
|
||||
.actions{display:flex;align-items:center;gap:1rem;flex-wrap:wrap;margin-bottom:1.5rem}
|
||||
button{padding:.55rem 1.3rem;border:none;border-radius:.5rem;cursor:pointer;font-weight:600;font-size:.85rem;transition:.15s}
|
||||
#btnSync{background:#2563eb;color:#fff}
|
||||
#btnSync:hover:not(:disabled){background:#1d4ed8}
|
||||
#btnSync:disabled{opacity:.4;cursor:not-allowed}
|
||||
#btnRefresh{background:#334155;color:#cbd5e1}
|
||||
#btnRefresh:hover{background:#475569}
|
||||
#nextSync{font-size:.8rem;color:#64748b;margin-left:auto}
|
||||
#msg{font-size:.82rem;padding:.35rem .75rem;border-radius:.4rem;display:none}
|
||||
#msg.ok{background:#14532d;color:#86efac;display:inline-block}
|
||||
#msg.err{background:#450a0a;color:#fca5a5;display:inline-block}
|
||||
#progressBox{background:#1e293b;border:1px solid #334155;border-radius:.75rem;padding:1rem 1.25rem;margin-bottom:1.5rem;display:none}
|
||||
#progressBox.visible{display:block}
|
||||
.progress-header{display:flex;justify-content:space-between;margin-bottom:.6rem;font-size:.85rem}
|
||||
.progress-bar-bg{background:#0f172a;border-radius:9999px;height:12px;overflow:hidden}
|
||||
.progress-bar-fill{height:100%;background:linear-gradient(90deg,#059669,#2563eb);border-radius:9999px;transition:width .5s ease}
|
||||
.progress-detail{margin-top:.5rem;font-size:.78rem;color:#64748b}
|
||||
.grid{display:grid;grid-template-columns:1fr 1fr;gap:1.5rem}
|
||||
@media(max-width:700px){.grid{grid-template-columns:1fr}}
|
||||
.card{background:#1e293b;border:1px solid #334155;border-radius:.75rem;overflow:hidden}
|
||||
.card-header{padding:.65rem 1rem;background:#0f172a;border-bottom:1px solid #334155;font-size:.75rem;font-weight:600;text-transform:uppercase;letter-spacing:.05em;color:#94a3b8}
|
||||
table{width:100%;border-collapse:collapse;font-size:.85rem}
|
||||
th{padding:.55rem 1rem;text-align:left;color:#64748b;font-weight:500;border-bottom:1px solid #334155}
|
||||
td{padding:.55rem 1rem;border-bottom:1px solid #1e293b}
|
||||
tr:last-child td{border-bottom:none}
|
||||
.ok{color:#4ade80}.err{color:#f87171}
|
||||
.log-box{height:320px;overflow-y:auto;font-family:monospace;font-size:.76rem;padding:.75rem 1rem;background:#0a0f1e;color:#a5f3fc;line-height:1.7;user-select:text;cursor:text}
|
||||
.log-line{white-space:pre-wrap;word-break:break-all}
|
||||
.log-line.error{color:#f87171}
|
||||
.ts{color:#475569;margin-right:.5rem}
|
||||
#btnCopyLogs{background:#334155;color:#cbd5e1;font-size:.75rem;padding:.35rem .8rem}
|
||||
#btnCopyLogs:hover{background:#475569}
|
||||
#lastRefresh{font-size:.75rem;color:#475569}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>⚡ Sync Hostinger</h1>
|
||||
<span class="subtitle">PG Debian → PG Hostinger</span>
|
||||
<span id="statusBadge" class="badge idle">Idle</span>
|
||||
<span id="lastRefresh" style="margin-left:auto"></span>
|
||||
</header>
|
||||
<main>
|
||||
<div class="actions">
|
||||
<button id="btnSync" onclick="runSync()">▶ Lancer sync maintenant</button>
|
||||
<button id="btnRefresh" onclick="refresh()">↺ Rafraîchir</button>
|
||||
<span id="msg"></span>
|
||||
<span id="nextSync"></span>
|
||||
</div>
|
||||
|
||||
<div id="progressBox">
|
||||
<div class="progress-header">
|
||||
<span id="progressLabel">Synchronisation en cours…</span>
|
||||
<span id="progressPct" style="font-weight:700;color:#93c5fd">0%</span>
|
||||
</div>
|
||||
<div class="progress-bar-bg">
|
||||
<div class="progress-bar-fill" id="progressBar" style="width:0%"></div>
|
||||
</div>
|
||||
<div class="progress-detail" id="progressDetail"></div>
|
||||
</div>
|
||||
|
||||
<div class="grid">
|
||||
<div class="card" style="grid-column:1/-1">
|
||||
<div class="card-header">Statut des tables</div>
|
||||
<table>
|
||||
<thead><tr><th>Table</th><th>Dernière sync</th><th>Lignes</th><th>Statut</th></tr></thead>
|
||||
<tbody id="tableBody"><tr><td colspan="4" style="color:#475569;padding:1rem">Chargement…</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="card" style="grid-column:1/-1">
|
||||
<div class="card-header" style="display:flex;justify-content:space-between;align-items:center">
|
||||
<span>Logs</span>
|
||||
<button id="btnCopyLogs" onclick="copyLogs()">Copier les logs</button>
|
||||
</div>
|
||||
<div class="log-box" id="logBox">Chargement…</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
<script>
|
||||
function fmtDate(iso){
|
||||
if(!iso) return '—';
|
||||
return new Date(iso).toLocaleString('fr-FR');
|
||||
}
|
||||
function renderProgress(progress, running){
|
||||
const box = document.getElementById('progressBox');
|
||||
if(!running || !progress){ box.classList.remove('visible'); return; }
|
||||
box.classList.add('visible');
|
||||
document.getElementById('progressBar').style.width = progress.pct + '%';
|
||||
document.getElementById('progressPct').textContent = progress.pct + '%';
|
||||
document.getElementById('progressLabel').textContent =
|
||||
'Sync en cours — table ' + progress.index + '/' + progress.total;
|
||||
const elapsed = Math.round((Date.now() - new Date(progress.startedAt)) / 1000);
|
||||
document.getElementById('progressDetail').textContent =
|
||||
'En cours : ' + progress.current + ' | Durée : ' + elapsed + 's';
|
||||
}
|
||||
function renderStatus(data){
|
||||
const badge = document.getElementById('statusBadge');
|
||||
badge.textContent = data.running ? 'En cours…' : 'Idle';
|
||||
badge.className = 'badge ' + (data.running ? 'running' : 'idle');
|
||||
document.getElementById('btnSync').disabled = data.running;
|
||||
renderProgress(data.progress, data.running);
|
||||
if(!data.tables.length){
|
||||
document.getElementById('tableBody').innerHTML =
|
||||
'<tr><td colspan="4" style="color:#475569;padding:1rem">Aucune sync — cliquez sur Lancer sync maintenant</td></tr>';
|
||||
return;
|
||||
}
|
||||
document.getElementById('tableBody').innerHTML = data.tables.map(t => \`
|
||||
<tr>
|
||||
<td><strong>\${t.table_name}</strong></td>
|
||||
<td>\${fmtDate(t.last_sync)}</td>
|
||||
<td>\${t.rows_synced ?? '—'}</td>
|
||||
<td class="\${t.status==='ok'?'ok':'err'}">\${t.status ?? '—'}\${t.error_msg?' — '+t.error_msg:''}</td>
|
||||
</tr>
|
||||
\`).join('');
|
||||
}
|
||||
function renderLogs(logs){
|
||||
const box = document.getElementById('logBox');
|
||||
if(!logs.length){ box.textContent = 'Aucun log.'; return; }
|
||||
const atBottom = box.scrollTop + box.clientHeight >= box.scrollHeight - 10;
|
||||
box.innerHTML = logs.map(l => \`
|
||||
<div class="log-line \${l.level}">
|
||||
<span class="ts">\${new Date(l.ts).toLocaleTimeString('fr-FR')}</span>\${l.msg}
|
||||
</div>
|
||||
\`).join('');
|
||||
if(atBottom) box.scrollTop = box.scrollHeight;
|
||||
}
|
||||
async function refresh(){
|
||||
try{
|
||||
const [s, logs, next] = await Promise.all([
|
||||
fetch('/api/status').then(r=>r.json()),
|
||||
fetch('/api/logs?limit=150').then(r=>r.json()),
|
||||
fetch('/api/next').then(r=>r.json()),
|
||||
]);
|
||||
renderStatus(s);
|
||||
renderLogs(logs);
|
||||
document.getElementById('nextSync').textContent = 'Prochaine sync auto : ' + fmtDate(next.next);
|
||||
document.getElementById('lastRefresh').textContent = 'Mis à jour ' + new Date().toLocaleTimeString('fr-FR');
|
||||
} catch(e){ console.error(e); }
|
||||
}
|
||||
function copyLogs(){
|
||||
const box = document.getElementById('logBox');
|
||||
const text = Array.from(box.querySelectorAll('.log-line')).map(l => l.textContent).join('\\n');
|
||||
const btn = document.getElementById('btnCopyLogs');
|
||||
function onSuccess(){ btn.textContent = 'Copié !'; setTimeout(() => { btn.textContent = 'Copier les logs'; }, 2000); }
|
||||
if(navigator.clipboard && window.isSecureContext){
|
||||
navigator.clipboard.writeText(text).then(onSuccess).catch(fallback);
|
||||
} else { fallback(); }
|
||||
function fallback(){
|
||||
const ta = document.createElement('textarea');
|
||||
ta.value = text;
|
||||
ta.style.cssText = 'position:fixed;top:-9999px;left:-9999px;opacity:0';
|
||||
document.body.appendChild(ta); ta.focus(); ta.select();
|
||||
try{ document.execCommand('copy'); onSuccess(); } catch(e){ alert('Sélectionnez les logs et Ctrl+C'); }
|
||||
document.body.removeChild(ta);
|
||||
}
|
||||
}
|
||||
async function runSync(){
|
||||
const msg = document.getElementById('msg');
|
||||
msg.style.display = 'none';
|
||||
const r = await fetch('/api/sync/run',{method:'POST'});
|
||||
const data = await r.json();
|
||||
msg.textContent = r.ok ? '✓ ' + data.message : '✗ ' + data.error;
|
||||
msg.className = r.ok ? 'ok' : 'err';
|
||||
setTimeout(()=>{ msg.style.display='none'; }, 5000);
|
||||
setTimeout(refresh, 800);
|
||||
}
|
||||
let timer;
|
||||
async function loop(){
|
||||
await refresh();
|
||||
const s = await fetch('/api/status').then(r=>r.json()).catch(()=>({}));
|
||||
timer = setTimeout(loop, s.running ? 3000 : 15000);
|
||||
}
|
||||
loop();
|
||||
</script>
|
||||
</body>
|
||||
</html>`;
|
||||
module.exports = { app, startDashboard, registerSyncFn, registerNextRunFn };
|
||||
+191
@@ -0,0 +1,191 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Fonctions pures de génération SQL (aucun accès base) — testées par test/sql.test.js.
|
||||
* Toutes les tables sont dans le schéma public, côté source comme côté destination.
|
||||
*/
|
||||
|
||||
/** Quote un identifiant PostgreSQL ("nom" avec doublement des guillemets). */
|
||||
function qi(name) {
|
||||
return '"' + String(name).replace(/"/g, '""') + '"';
|
||||
}
|
||||
|
||||
/** Nom qualifié public."table". */
|
||||
function qt(table) {
|
||||
return 'public.' + qi(table);
|
||||
}
|
||||
|
||||
/** Littéral SQL texte (échappement des apostrophes). */
|
||||
function lit(value) {
|
||||
return "'" + String(value).replace(/'/g, "''") + "'";
|
||||
}
|
||||
|
||||
const DATE_RE = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
/** Littéral date 'YYYY-MM-DD' strictement validé (utilisé dans les DECLARE CURSOR). */
|
||||
function dateLit(value) {
|
||||
if (!DATE_RE.test(String(value))) throw new Error(`Date invalide : ${value}`);
|
||||
return lit(value);
|
||||
}
|
||||
|
||||
/** Liste de colonnes quotées, dans l'ordre fourni. */
|
||||
function colList(cols) {
|
||||
return cols.map(c => qi(c.name ?? c)).join(', ');
|
||||
}
|
||||
|
||||
/**
|
||||
* CREATE TABLE IF NOT EXISTS reproduisant colonnes (ordre, type exact, NOT NULL) et clé primaire.
|
||||
* Pas de DEFAULT (les séquences ne sont pas répliquées) ni de clé étrangère.
|
||||
* cols : [{ name, type, notnull }], pk : ['col', ...]
|
||||
*/
|
||||
function buildCreateTable(table, cols, pk = []) {
|
||||
const defs = cols.map(c => ` ${qi(c.name)} ${c.type}${c.notnull ? ' NOT NULL' : ''}`);
|
||||
if (pk.length) defs.push(` PRIMARY KEY (${pk.map(qi).join(', ')})`);
|
||||
return `CREATE TABLE IF NOT EXISTS ${qt(table)} (\n${defs.join(',\n')}\n)`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ajout d'une colonne manquante. Toujours NULLable : un NOT NULL sans DEFAULT
|
||||
* échouerait sur une table déjà remplie ; la colonne est remplie par la copie suivante.
|
||||
*/
|
||||
function buildAddColumn(table, col) {
|
||||
return `ALTER TABLE ${qt(table)} ADD COLUMN IF NOT EXISTS ${qi(col.name)} ${col.type}`;
|
||||
}
|
||||
|
||||
function buildAddPrimaryKey(table, pk) {
|
||||
return `ALTER TABLE ${qt(table)} ADD PRIMARY KEY (${pk.map(qi).join(', ')})`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Transforme un pg_indexes.indexdef ("CREATE [UNIQUE] INDEX nom ON public.t USING ...")
|
||||
* en version idempotente avec IF NOT EXISTS (même nom d'index).
|
||||
*/
|
||||
function indexDefIfNotExists(indexdef) {
|
||||
const out = String(indexdef).replace(/^CREATE (UNIQUE )?INDEX (?!IF NOT EXISTS )/i, (m, u) => `CREATE ${u || ''}INDEX IF NOT EXISTS `);
|
||||
if (out === indexdef && !/^CREATE (UNIQUE )?INDEX IF NOT EXISTS /i.test(indexdef)) {
|
||||
throw new Error(`Définition d'index inattendue : ${indexdef}`);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** L'index nécessite-t-il l'extension pg_trgm ? */
|
||||
function needsTrgm(indexdef) {
|
||||
return /\b(gin|gist)_trgm_ops\b/i.test(String(indexdef));
|
||||
}
|
||||
|
||||
/**
|
||||
* Requête de signature par mois : nb de lignes + somme des hash de chaque ligne.
|
||||
* La liste de colonnes est explicite (ordre source) pour que source et destination
|
||||
* hachent exactement les mêmes colonnes, même si la destination a des colonnes en plus.
|
||||
* Résultat : m ('YYYY-MM-DD' ou NULL), n (texte), h (texte).
|
||||
*/
|
||||
function buildSignatureQuery(table, cols, dateCol, where = null) {
|
||||
const d = qi(dateCol);
|
||||
return `SELECT to_char(date_trunc('month', ${d}), 'YYYY-MM-DD') AS m,
|
||||
count(*)::text AS n,
|
||||
coalesce(sum(hashtext(ROW(${colList(cols)})::text)::bigint), 0)::text AS h
|
||||
FROM ${qt(table)}${where ? `\nWHERE ${where}` : ''}
|
||||
GROUP BY 1`;
|
||||
}
|
||||
|
||||
const NULL_MONTH = 'NULL';
|
||||
|
||||
/** Convertit le résultat d'une requête de signature en Map mois → "n:h". */
|
||||
function signatureMap(rows) {
|
||||
const map = new Map();
|
||||
for (const r of rows) map.set(r.m ?? NULL_MONTH, `${r.n}:${r.h}`);
|
||||
return map;
|
||||
}
|
||||
|
||||
/** Total des lignes d'une Map de signatures. */
|
||||
function signatureTotal(map) {
|
||||
let total = 0;
|
||||
for (const v of map.values()) total += Number(v.split(':')[0]);
|
||||
return total;
|
||||
}
|
||||
|
||||
/** Mois dont la signature diffère, ou présents d'un seul côté (triés, NULL en premier). */
|
||||
function diffMonths(srcMap, dstMap) {
|
||||
const keys = new Set([...srcMap.keys(), ...dstMap.keys()]);
|
||||
return [...keys].filter(k => srcMap.get(k) !== dstMap.get(k)).sort();
|
||||
}
|
||||
|
||||
/** 1er jour du mois suivant ('YYYY-MM-01' → 'YYYY-MM-01'). */
|
||||
function nextMonth(month) {
|
||||
if (!DATE_RE.test(month)) throw new Error(`Mois invalide : ${month}`);
|
||||
const [y, m] = month.split('-').map(Number);
|
||||
const ny = m === 12 ? y + 1 : y;
|
||||
const nm = m === 12 ? 1 : m + 1;
|
||||
return `${ny}-${String(nm).padStart(2, '0')}-01`;
|
||||
}
|
||||
|
||||
/** Condition WHERE sélectionnant un mois (ou les dates NULL). */
|
||||
function monthCondition(dateCol, month) {
|
||||
const d = qi(dateCol);
|
||||
if (month === NULL_MONTH) return `${d} IS NULL`;
|
||||
return `${d} >= ${dateLit(month)} AND ${d} < ${dateLit(nextMonth(month))}`;
|
||||
}
|
||||
|
||||
/** Condition WHERE de la fenêtre glissante (date >= cutoff). */
|
||||
function recentCondition(dateCol, cutoff) {
|
||||
return `${qi(dateCol)} >= ${dateLit(cutoff)}`;
|
||||
}
|
||||
|
||||
/** Taille de lot d'INSERT : limite de 65535 paramètres par requête. */
|
||||
function insertChunkSize(ncols) {
|
||||
return Math.max(1, Math.floor(60000 / ncols));
|
||||
}
|
||||
|
||||
/** INSERT multi-lignes paramétré ($1..$n). */
|
||||
function buildInsert(table, cols, nrows) {
|
||||
const n = cols.length;
|
||||
const tuples = new Array(nrows);
|
||||
for (let r = 0; r < nrows; r++) {
|
||||
const ph = new Array(n);
|
||||
for (let c = 0; c < n; c++) ph[c] = '$' + (r * n + c + 1);
|
||||
tuples[r] = '(' + ph.join(',') + ')';
|
||||
}
|
||||
return `INSERT INTO ${qt(table)} (${colList(cols)}) VALUES ${tuples.join(',')}`;
|
||||
}
|
||||
|
||||
/** SELECT des colonnes source (ordre source) avec filtre optionnel. */
|
||||
function buildSelect(table, cols, where = null) {
|
||||
return `SELECT ${colList(cols)} FROM ${qt(table)}${where ? ` WHERE ${where}` : ''}`;
|
||||
}
|
||||
|
||||
/** "a,b , c" → ['a','b','c'] (espaces retirés, éléments vides ignorés). */
|
||||
function parseList(value) {
|
||||
return String(value || '')
|
||||
.split(',')
|
||||
.map(s => s.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
/** "mvtart:datmvt,meteo:date" → { mvtart: 'datmvt', meteo: 'date' }. */
|
||||
function parseWindowTables(value) {
|
||||
const out = {};
|
||||
for (const item of parseList(value)) {
|
||||
const [table, col] = item.split(':').map(s => (s || '').trim());
|
||||
if (!table || !col) throw new Error(`WINDOW_TABLES invalide : "${item}" (attendu table:colonne)`);
|
||||
out[table] = col;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Date 'YYYY-MM-DD' (fuseau donné) de "aujourd'hui - days". */
|
||||
function cutoffDate(days, timeZone = 'Europe/Paris', now = new Date()) {
|
||||
const fmt = new Intl.DateTimeFormat('en-CA', { timeZone, year: 'numeric', month: '2-digit', day: '2-digit' });
|
||||
const today = fmt.format(now); // en-CA → YYYY-MM-DD
|
||||
const [y, m, d] = today.split('-').map(Number);
|
||||
const t = new Date(Date.UTC(y, m - 1, d - days));
|
||||
return t.toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
qi, qt, lit, dateLit, colList,
|
||||
buildCreateTable, buildAddColumn, buildAddPrimaryKey,
|
||||
indexDefIfNotExists, needsTrgm,
|
||||
buildSignatureQuery, signatureMap, signatureTotal, diffMonths, NULL_MONTH,
|
||||
nextMonth, monthCondition, recentCondition,
|
||||
insertChunkSize, buildInsert, buildSelect,
|
||||
parseList, parseWindowTables, cutoffDate,
|
||||
};
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
'use strict';
|
||||
/**
|
||||
* Orchestration d'une exécution : verrou, attente de la sync Debian, miroir du schéma,
|
||||
* copie table par table, journalisation dans sync_log (destination), alerte.
|
||||
*/
|
||||
const { config } = require('./config');
|
||||
const log = require('./logger');
|
||||
const { getSrc, getDst } = require('./db');
|
||||
const { readCatalog, mirrorTable, ensureTrgm, mirrorIndexes, ensureDstSyncLog } = require('./schema');
|
||||
const { copyFull, copyWindow } = require('./copy');
|
||||
const { sendAlert } = require('./alert');
|
||||
const { qt } = require('./sql');
|
||||
|
||||
const SOURCE_SYNC_LOG = 'sync_log'; // table source traitée à part
|
||||
const DST_SOURCE_SYNC_LOG = 'source_sync_log';
|
||||
|
||||
// État partagé avec le dashboard
|
||||
const state = {
|
||||
running: false,
|
||||
force: false,
|
||||
phase: null, // 'verrou' | 'attente Debian' | 'schéma' | 'copie' | null
|
||||
progress: null, // { current, index, total, startedAt }
|
||||
lastRun: null, // { startedAt, finishedAt, force, status, tables: [...] }
|
||||
};
|
||||
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
async function writeLog(dst, entry) {
|
||||
await dst.query(
|
||||
`INSERT INTO ${qt('sync_log')} (table_name, last_sync, rows_synced, status, error_msg, strategy, duration_ms, detail)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
last_sync = EXCLUDED.last_sync, rows_synced = EXCLUDED.rows_synced, status = EXCLUDED.status,
|
||||
error_msg = EXCLUDED.error_msg, strategy = EXCLUDED.strategy,
|
||||
duration_ms = EXCLUDED.duration_ms, detail = EXCLUDED.detail`,
|
||||
// Date JS : stockée correctement en TIMESTAMP (heure locale du conteneur, TZ=Europe/Paris) comme en TIMESTAMPTZ
|
||||
[entry.table, new Date(), Math.min(entry.rows ?? 0, 2147483647), entry.status, entry.error ?? null,
|
||||
entry.strategy ?? null, entry.durationMs ?? null, entry.detail ?? null],
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attend que la sync Debian soit terminée (ligne sync_log '_run' ≠ 'running').
|
||||
* Retourne 'ok' | 'absent' | 'timeout'.
|
||||
*/
|
||||
async function waitForDebian() {
|
||||
const src = getSrc();
|
||||
const deadline = Date.now() + config.waitMaxMs;
|
||||
let first = true;
|
||||
for (;;) {
|
||||
let status;
|
||||
try {
|
||||
const { rows } = await src.query(`SELECT status FROM ${qt(SOURCE_SYNC_LOG)} WHERE table_name = '_run'`);
|
||||
if (!rows.length) return 'absent';
|
||||
status = rows[0].status;
|
||||
} catch (err) {
|
||||
if (err.code === '42P01') return 'absent'; // sync_log inexistante côté source
|
||||
throw err;
|
||||
}
|
||||
if (status !== 'running') return 'ok';
|
||||
if (Date.now() >= deadline) return 'timeout';
|
||||
if (first) log.info(`Sync Debian en cours — attente (contrôle toutes les ${config.waitPollMs / 60000} min, max ${config.waitMaxMs / 60000} min)`);
|
||||
first = false;
|
||||
await sleep(Math.min(config.waitPollMs, Math.max(1000, deadline - Date.now())));
|
||||
}
|
||||
}
|
||||
|
||||
/** Liste ordonnée des copies à effectuer : [{ src, dstName, strategy, dateCol }]. */
|
||||
function planTables(srcCatalog) {
|
||||
const plan = [];
|
||||
for (const name of [...srcCatalog.keys()].sort()) {
|
||||
if (name === SOURCE_SYNC_LOG || config.excludeTables.has(name)) continue;
|
||||
const t = srcCatalog.get(name);
|
||||
const dateCol = config.windowTables[name];
|
||||
if (dateCol && t.columns.some(c => c.name === dateCol)) {
|
||||
plan.push({ src: t, dstName: name, strategy: 'window', dateCol });
|
||||
} else {
|
||||
if (dateCol) log.warn(`[${name}] colonne ${dateCol} absente de la source — copie complète`);
|
||||
plan.push({ src: t, dstName: name, strategy: 'full' });
|
||||
}
|
||||
}
|
||||
if (srcCatalog.has(SOURCE_SYNC_LOG) && !config.excludeTables.has(SOURCE_SYNC_LOG)) {
|
||||
plan.push({ src: srcCatalog.get(SOURCE_SYNC_LOG), dstName: DST_SOURCE_SYNC_LOG, strategy: 'full' });
|
||||
}
|
||||
return plan;
|
||||
}
|
||||
|
||||
async function syncOne(item, ctx) {
|
||||
const { src: srcTable, dstName, strategy, dateCol } = item;
|
||||
const started = Date.now();
|
||||
const entry = { table: dstName, strategy, rows: 0, status: 'ok', error: null, detail: null };
|
||||
const srcClient = await getSrc().connect();
|
||||
const dstClient = await getDst().connect();
|
||||
try {
|
||||
const warnings = await mirrorTable(dstClient, srcTable, dstName, ctx.dstCatalog);
|
||||
// Instantané cohérent de la source pour toute la table (comptage, signatures, curseurs)
|
||||
await srcClient.query('BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY');
|
||||
try {
|
||||
const r = strategy === 'window'
|
||||
? await copyWindow(srcClient, dstClient, srcTable.name, srcTable.columns, dateCol, ctx.force)
|
||||
: await copyFull(srcClient, dstClient, srcTable.name, dstName, srcTable.columns, ctx.force);
|
||||
entry.rows = r.rows;
|
||||
entry.detail = r.detail;
|
||||
} finally {
|
||||
await srcClient.query('COMMIT').catch(() => {});
|
||||
}
|
||||
warnings.push(...await mirrorIndexes(dstClient, srcTable, dstName, ctx.dstCatalog, ctx.trgmOk));
|
||||
if (warnings.length) entry.detail = `${entry.detail} | ${warnings.length} avertissement(s) : ${warnings.join(' ; ')}`;
|
||||
log.info(`[${dstName}] OK — ${entry.detail}`);
|
||||
} catch (err) {
|
||||
entry.status = 'error';
|
||||
entry.error = err.message;
|
||||
log.error(`[${dstName}] ERREUR : ${err.message}`);
|
||||
} finally {
|
||||
srcClient.release();
|
||||
dstClient.release();
|
||||
}
|
||||
entry.durationMs = Date.now() - started;
|
||||
await writeLog(getDst(), entry).catch(e => log.error(`[${dstName}] écriture sync_log impossible : ${e.message}`));
|
||||
return entry;
|
||||
}
|
||||
|
||||
/**
|
||||
* Lance une synchronisation complète.
|
||||
* force = ignorer le garde-fou volume + rechargement complet des tables "fenêtre".
|
||||
* Retourne { skipped } | { status, tables }.
|
||||
*/
|
||||
async function runSync({ force = false, trigger = 'manuel' } = {}) {
|
||||
if (state.running) {
|
||||
log.warn('Sync déjà en cours dans ce process — demande ignorée');
|
||||
return { skipped: 'running' };
|
||||
}
|
||||
state.running = true;
|
||||
state.force = force;
|
||||
state.phase = 'verrou';
|
||||
state.progress = null;
|
||||
const startedAt = new Date();
|
||||
const results = [];
|
||||
let lockClient = null;
|
||||
let locked = false;
|
||||
let fatal = null;
|
||||
|
||||
try {
|
||||
lockClient = await getDst().connect();
|
||||
const { rows } = await lockClient.query('SELECT pg_try_advisory_lock($1::bigint) AS ok', [config.lockKey]);
|
||||
locked = rows[0].ok;
|
||||
if (!locked) {
|
||||
log.warn('Une autre synchronisation détient le verrou (autre process) — exécution ignorée');
|
||||
return { skipped: 'locked' };
|
||||
}
|
||||
log.info(`=== SYNC START (${trigger}${force ? ', FORCÉE' : ''}) ===`);
|
||||
await ensureDstSyncLog(lockClient);
|
||||
await writeLog(lockClient, { table: '_run', status: 'running', strategy: force ? 'force' : trigger, detail: `démarrée ${startedAt.toISOString()}` });
|
||||
|
||||
state.phase = 'attente Debian';
|
||||
const wait = await waitForDebian();
|
||||
if (wait === 'timeout') {
|
||||
const msg = `Sync Debian toujours "running" après ${config.waitMaxMs / 60000} min — copie lancée quand même`;
|
||||
log.error(msg);
|
||||
await sendAlert(`[ff_sync_hostinger] ${msg}`);
|
||||
}
|
||||
|
||||
state.phase = 'schéma';
|
||||
const srcCatalog = await readCatalog(getSrc());
|
||||
const dstCatalog = await readCatalog(lockClient);
|
||||
const trgmOk = await ensureTrgm(lockClient, srcCatalog);
|
||||
const plan = planTables(srcCatalog);
|
||||
log.info(`${plan.length} tables à synchroniser (${plan.filter(p => p.strategy === 'window').length} par fenêtre)`);
|
||||
|
||||
state.phase = 'copie';
|
||||
const ctx = { force, dstCatalog, trgmOk };
|
||||
for (let i = 0; i < plan.length; i++) {
|
||||
state.progress = { current: plan[i].dstName, index: i + 1, total: plan.length, startedAt: startedAt.toISOString() };
|
||||
results.push(await syncOne(plan[i], ctx));
|
||||
}
|
||||
} catch (err) {
|
||||
fatal = err;
|
||||
log.error(`ERREUR FATALE : ${err.message}`);
|
||||
} finally {
|
||||
const failed = results.filter(r => r.status !== 'ok');
|
||||
if (locked) {
|
||||
const status = fatal || failed.length ? 'error' : 'ok';
|
||||
const detail = fatal
|
||||
? `erreur fatale : ${fatal.message}`
|
||||
: `${results.length} tables, ${failed.length} en erreur`;
|
||||
await writeLog(lockClient, {
|
||||
table: '_run', status, strategy: force ? 'force' : trigger,
|
||||
rows: results.reduce((s, r) => s + (r.rows || 0), 0),
|
||||
error: fatal ? fatal.message : failed.length ? failed.map(f => f.table).join(', ') : null,
|
||||
durationMs: Date.now() - startedAt.getTime(), detail,
|
||||
}).catch(e => log.error(`écriture _run impossible : ${e.message}`));
|
||||
await lockClient.query('SELECT pg_advisory_unlock($1::bigint)', [config.lockKey]).catch(() => {});
|
||||
if (fatal || failed.length) {
|
||||
const lines = failed.slice(0, 15).map(f => `• ${f.table} : ${f.error}`);
|
||||
await sendAlert(
|
||||
`[ff_sync_hostinger] Sync ${fatal ? 'en échec' : 'terminée avec erreurs'} — ${detail}` +
|
||||
(lines.length ? `\n${lines.join('\n')}` : ''),
|
||||
);
|
||||
}
|
||||
log.info(`=== SYNC FIN en ${((Date.now() - startedAt.getTime()) / 1000).toFixed(1)} s — ${detail} ===`);
|
||||
state.lastRun = {
|
||||
startedAt: startedAt.toISOString(), finishedAt: new Date().toISOString(),
|
||||
force, trigger, status: fatal || failed.length ? 'error' : 'ok', tables: results,
|
||||
};
|
||||
} else if (fatal) {
|
||||
await sendAlert(`[ff_sync_hostinger] Sync impossible : ${fatal.message}`);
|
||||
}
|
||||
if (lockClient) lockClient.release();
|
||||
state.running = false;
|
||||
state.phase = null;
|
||||
state.progress = null;
|
||||
}
|
||||
return { status: fatal || results.some(r => r.status !== 'ok') ? 'error' : 'ok', tables: results };
|
||||
}
|
||||
|
||||
module.exports = { runSync, state, planTables, waitForDebian };
|
||||
@@ -1,20 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Sync PostgreSQL Debian → PostgreSQL Hostinger
|
||||
# Source : frps:15432 (tunnel frp vers postgres Debian)
|
||||
# Destination : postgres_db (container existant sur Hostinger)
|
||||
|
||||
set -e
|
||||
|
||||
echo "=== SYNC START $(date) ==="
|
||||
|
||||
PGPASSWORD=$SRC_PASS pg_dump \
|
||||
-h "$SRC_HOST" -p "$SRC_PORT" \
|
||||
-U "$SRC_USER" -d "$SRC_DB" \
|
||||
--data-only --if-exists --clean \
|
||||
--no-acl --no-owner \
|
||||
| PGPASSWORD=$DST_PASS psql \
|
||||
-h "$DST_HOST" -p "$DST_PORT" \
|
||||
-U "$DST_USER" -d "$DST_DB" \
|
||||
--set ON_ERROR_STOP=0
|
||||
|
||||
echo "=== SYNC DONE $(date) ==="
|
||||
@@ -1,44 +0,0 @@
|
||||
'use strict';
|
||||
// Définition des tables à synchroniser PG Debian → PG Hostinger
|
||||
// pk: [] = full refresh (TRUNCATE + INSERT), sinon ON CONFLICT DO UPDATE
|
||||
|
||||
const TABLES = [
|
||||
// Référentiel
|
||||
{ name: 'nomenclature', pk: ['no_id'] },
|
||||
{ name: 'gammes', pk: ['no_id'] },
|
||||
{ name: 'saisons', pk: ['no_id'] },
|
||||
// Articles
|
||||
{ name: 'articles', pk: ['no_id'] },
|
||||
{ name: 'article_infosup', pk: ['artnoid'] },
|
||||
{ name: 'art_gtin', pk: ['idarticle', 'gtin'] },
|
||||
{ name: 'art_gamme_saison', pk: ['artnoid', 'idgamme', 'idsaison'] },
|
||||
// Fournisseurs
|
||||
{ name: 'fouadr1', pk: ['code', 'sit_code'] },
|
||||
{ name: 'artfou1', pk: ['no_id'] },
|
||||
{ name: 'artfou2', pk: ['idartfou1'] },
|
||||
// Stock & prix
|
||||
{ name: 'cube_pa', pk: ['artnoid'] },
|
||||
{ name: 'cube_pv', pk: ['artnoid', 'site'] },
|
||||
{ name: 'cube_stock', pk: ['artnoid', 'site'] },
|
||||
// Mouvements (sans PK → full refresh)
|
||||
{ name: 'mvtart', pk: [] },
|
||||
{ name: 'mvtreg', pk: [] },
|
||||
// Ranking
|
||||
{ name: 'ranking', pk: ['gencod', 'site'] },
|
||||
// CA journalier + trafic
|
||||
{ name: 'statopca', pk: ['site', 'datmvt'] },
|
||||
// Gamme conseillée par site
|
||||
{ name: 'stat_dispoperm', pk: ['codesite', 'code_art'] },
|
||||
// Journal de commande conseillé
|
||||
{ name: 'phenix_quantite_conseille', pk: ['ident'] },
|
||||
// Commandes
|
||||
{ name: 'cdefou_reception', pk: ['no_id'] },
|
||||
{ name: 'cdefou_vivant', pk: ['cdefou_ligne_com_no_id', 'artfou1_no_id'] },
|
||||
{ name: 'cdefou_receplig', pk: ['no_id'] },
|
||||
// Publicités
|
||||
{ name: 'pub_entetes', pk: ['ent_npub'] },
|
||||
{ name: 'pub_ecoulement', pk: ['site', 'tcr_code'] },
|
||||
{ name: 'pub_ecoulement_detail', pk: ['site', 'tcr_code', 'artnoid'] },
|
||||
];
|
||||
|
||||
module.exports = { TABLES };
|
||||
@@ -0,0 +1,97 @@
|
||||
'use strict';
|
||||
// Tests des fonctions pures de génération SQL : `npm test` (node --test, sans base de données).
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const s = require('../sql');
|
||||
|
||||
test('qi échappe les guillemets', () => {
|
||||
assert.equal(s.qi('mvtart'), '"mvtart"');
|
||||
assert.equal(s.qi('a"b'), '"a""b"');
|
||||
});
|
||||
|
||||
test('buildCreateTable : colonnes ordonnées, NOT NULL, PK', () => {
|
||||
const sql = s.buildCreateTable('mvtart', [
|
||||
{ name: 'no_id', type: 'bigint', notnull: true },
|
||||
{ name: 'datmvt', type: 'timestamp without time zone', notnull: false },
|
||||
{ name: 'qte', type: 'numeric(12,4)', notnull: false },
|
||||
], ['no_id']);
|
||||
assert.equal(sql,
|
||||
'CREATE TABLE IF NOT EXISTS public."mvtart" (\n' +
|
||||
' "no_id" bigint NOT NULL,\n' +
|
||||
' "datmvt" timestamp without time zone,\n' +
|
||||
' "qte" numeric(12,4),\n' +
|
||||
' PRIMARY KEY ("no_id")\n)');
|
||||
assert.ok(!s.buildCreateTable('t', [{ name: 'a', type: 'text' }], []).includes('PRIMARY KEY'));
|
||||
});
|
||||
|
||||
test('buildAddColumn est idempotent et nullable', () => {
|
||||
assert.equal(s.buildAddColumn('articles', { name: 'artcentrale', type: 'text', notnull: true }),
|
||||
'ALTER TABLE public."articles" ADD COLUMN IF NOT EXISTS "artcentrale" text');
|
||||
});
|
||||
|
||||
test('indexDefIfNotExists', () => {
|
||||
assert.equal(s.indexDefIfNotExists('CREATE INDEX idx_a ON public.articles USING btree (codein)'),
|
||||
'CREATE INDEX IF NOT EXISTS idx_a ON public.articles USING btree (codein)');
|
||||
assert.equal(s.indexDefIfNotExists('CREATE UNIQUE INDEX u ON public.t USING btree (a, b)'),
|
||||
'CREATE UNIQUE INDEX IF NOT EXISTS u ON public.t USING btree (a, b)');
|
||||
assert.equal(s.indexDefIfNotExists('CREATE INDEX IF NOT EXISTS x ON public.t (a)'),
|
||||
'CREATE INDEX IF NOT EXISTS x ON public.t (a)');
|
||||
assert.throws(() => s.indexDefIfNotExists('DROP TABLE x'));
|
||||
});
|
||||
|
||||
test('needsTrgm', () => {
|
||||
assert.ok(s.needsTrgm('CREATE INDEX i ON public.a USING gin (libelle1 gin_trgm_ops)'));
|
||||
assert.ok(!s.needsTrgm('CREATE INDEX i ON public.a USING btree (libelle1)'));
|
||||
});
|
||||
|
||||
test('buildSignatureQuery : colonnes explicites dans l\'ordre source', () => {
|
||||
const sql = s.buildSignatureQuery('mvtart', [{ name: 'no_id' }, { name: 'datmvt' }], 'datmvt');
|
||||
assert.match(sql, /date_trunc\('month', "datmvt"\)/);
|
||||
assert.match(sql, /hashtext\(ROW\("no_id", "datmvt"\)::text\)::bigint/);
|
||||
assert.match(sql, /FROM public\."mvtart"\nGROUP BY 1$/);
|
||||
});
|
||||
|
||||
test('signatures : diff, total, mois NULL', () => {
|
||||
const src = s.signatureMap([
|
||||
{ m: '2026-08-01', n: '10', h: '123' },
|
||||
{ m: '2026-09-01', n: '5', h: '7' },
|
||||
{ m: null, n: '1', h: '9' },
|
||||
]);
|
||||
const dst = s.signatureMap([
|
||||
{ m: '2026-08-01', n: '10', h: '123' },
|
||||
{ m: '2026-09-01', n: '5', h: '8' },
|
||||
{ m: '2026-07-01', n: '3', h: '1' },
|
||||
]);
|
||||
assert.deepEqual(s.diffMonths(src, dst), ['2026-07-01', '2026-09-01', s.NULL_MONTH]);
|
||||
assert.equal(s.signatureTotal(src), 16);
|
||||
});
|
||||
|
||||
test('monthCondition / nextMonth / recentCondition', () => {
|
||||
assert.equal(s.nextMonth('2026-12-01'), '2027-01-01');
|
||||
assert.equal(s.nextMonth('2026-02-01'), '2026-03-01');
|
||||
assert.equal(s.monthCondition('datmvt', '2026-12-01'), `"datmvt" >= '2026-12-01' AND "datmvt" < '2027-01-01'`);
|
||||
assert.equal(s.monthCondition('datmvt', s.NULL_MONTH), '"datmvt" IS NULL');
|
||||
assert.equal(s.recentCondition('date', '2026-10-07'), `"date" >= '2026-10-07'`);
|
||||
assert.throws(() => s.monthCondition('d', "2026-01-01'; DROP TABLE x; --"));
|
||||
});
|
||||
|
||||
test('buildInsert / insertChunkSize', () => {
|
||||
assert.equal(s.buildInsert('t', [{ name: 'a' }, { name: 'b' }], 2),
|
||||
'INSERT INTO public."t" ("a", "b") VALUES ($1,$2),($3,$4)');
|
||||
assert.equal(s.insertChunkSize(30), 2000);
|
||||
assert.equal(s.insertChunkSize(100000), 1);
|
||||
assert.ok(s.insertChunkSize(7) * 7 <= 65535);
|
||||
});
|
||||
|
||||
test('parseWindowTables / parseList', () => {
|
||||
assert.deepEqual(s.parseWindowTables('mvtart:datmvt, meteo:date'), { mvtart: 'datmvt', meteo: 'date' });
|
||||
assert.throws(() => s.parseWindowTables('mvtart'));
|
||||
assert.deepEqual(s.parseList(' a, ,b '), ['a', 'b']);
|
||||
assert.deepEqual(s.parseList(undefined), []);
|
||||
});
|
||||
|
||||
test('cutoffDate en Europe/Paris', () => {
|
||||
// 17/10/2026 23:30 UTC = 18/10/2026 01:30 à Paris
|
||||
assert.equal(s.cutoffDate(10, 'Europe/Paris', new Date('2026-10-17T23:30:00Z')), '2026-10-08');
|
||||
assert.equal(s.cutoffDate(0, 'Europe/Paris', new Date('2026-03-01T12:00:00Z')), '2026-03-01');
|
||||
});
|
||||
@@ -1,89 +0,0 @@
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Upsert en masse dans PG destination (chunks de 500 lignes).
|
||||
* Les colonnes sont déduites dynamiquement des lignes source.
|
||||
*/
|
||||
async function batchUpsert(pg, table, rows, pk) {
|
||||
if (!rows.length) return 0;
|
||||
const cols = Object.keys(rows[0]);
|
||||
const CHUNK = 500;
|
||||
let total = 0;
|
||||
|
||||
for (let i = 0; i < rows.length; i += CHUNK) {
|
||||
const chunk = rows.slice(i, i + CHUNK);
|
||||
const values = [];
|
||||
const placeholders = chunk.map((row, ri) => {
|
||||
const ph = cols.map((col, ci) => {
|
||||
values.push(row[col] ?? null);
|
||||
return `$${ri * cols.length + ci + 1}`;
|
||||
});
|
||||
return `(${ph.join(', ')})`;
|
||||
});
|
||||
const updateCols = cols.filter(c => !pk.includes(c));
|
||||
const updateSet = updateCols.length
|
||||
? updateCols.map(c => `${c} = EXCLUDED.${c}`).join(', ')
|
||||
: `${pk[0]} = EXCLUDED.${pk[0]}`; // fallback si toutes les cols sont PK
|
||||
|
||||
const sql = `
|
||||
INSERT INTO ${table} (${cols.join(', ')})
|
||||
VALUES ${placeholders.join(', ')}
|
||||
ON CONFLICT (${pk.join(', ')}) DO UPDATE SET ${updateSet}
|
||||
`;
|
||||
await pg.query(sql, values);
|
||||
total += chunk.length;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full refresh : TRUNCATE puis INSERT en masse.
|
||||
*/
|
||||
async function fullRefresh(pg, table, rows) {
|
||||
await pg.query(`TRUNCATE TABLE ${table} CASCADE`);
|
||||
if (!rows.length) return 0;
|
||||
|
||||
const cols = Object.keys(rows[0]);
|
||||
const CHUNK = 500;
|
||||
let total = 0;
|
||||
|
||||
for (let i = 0; i < rows.length; i += CHUNK) {
|
||||
const chunk = rows.slice(i, i + CHUNK);
|
||||
const values = [];
|
||||
const placeholders = chunk.map((row, ri) => {
|
||||
const ph = cols.map((col, ci) => {
|
||||
values.push(row[col] ?? null);
|
||||
return `$${ri * cols.length + ci + 1}`;
|
||||
});
|
||||
return `(${ph.join(', ')})`;
|
||||
});
|
||||
await pg.query(
|
||||
`INSERT INTO ${table} (${cols.join(', ')}) VALUES ${placeholders.join(', ')}`,
|
||||
values
|
||||
);
|
||||
total += chunk.length;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
async function logSync(pg, tableName, rowsSynced, status, errorMsg = null) {
|
||||
await pg.query(`
|
||||
INSERT INTO sync_log (table_name, last_sync, rows_synced, status, error_msg)
|
||||
VALUES ($1, NOW(), $2, $3, $4)
|
||||
ON CONFLICT (table_name) DO UPDATE SET
|
||||
last_sync = NOW(),
|
||||
rows_synced = $2,
|
||||
status = $3,
|
||||
error_msg = $4
|
||||
`, [tableName, rowsSynced, status, errorMsg]);
|
||||
}
|
||||
|
||||
async function getLastSync(pg, tableName) {
|
||||
const res = await pg.query(
|
||||
`SELECT last_sync FROM sync_log WHERE table_name = $1 AND status = 'ok'`,
|
||||
[tableName]
|
||||
);
|
||||
return res.rows[0]?.last_sync || null;
|
||||
}
|
||||
|
||||
module.exports = { batchUpsert, fullRefresh, logSync, getLastSync };
|
||||
Reference in new issue
Block a user