diff --git a/scripts/test-qlik-date-range.ts b/scripts/test-qlik-date-range.ts new file mode 100644 index 0000000..6efccbd --- /dev/null +++ b/scripts/test-qlik-date-range.ts @@ -0,0 +1,114 @@ +/** + * Test CLI pour `src/lib/qlik-date-range.ts` (aucune dépendance Next.js). + * Usage : npx tsx scripts/test-qlik-date-range.ts + * + * Vérifie : + * - les repères Qlik (2025-01-01→45658, etc.) + * - la fenêtre 12 mois complets pour `now=2026-06-21` (doit donner + * dateDebut=2025-06-01, dateFin=2026-05-31, qStart=45809, qEnd=46173, + * label=2025-06_2026-05, 365 serials) + * - exemples supplémentaires (1er du mois, fin d'année, année bissextile) + * - erreurs attendues sur entrée mal formée + * + * Aucun framework : assertions manuelles, exit code 0/1. + */ + +import { + isoDateToQlikSerial, + qlikSerialToIsoDate, + buildGridNetworkQlikDateFilter, + chunkSerials, + QLIK_DATE_ANCHORS, +} from "../src/lib/qlik-date-range"; + +let pass = 0; +let fail = 0; + +function eq(label: string, got: unknown, want: unknown): void { + const ok = JSON.stringify(got) === JSON.stringify(want); + if (ok) { + pass++; + console.log(` \u2713 ${label}`); + } else { + fail++; + console.log(` \u2717 ${label}\n got: ${JSON.stringify(got)}\n want: ${JSON.stringify(want)}`); + } +} + +function truthy(label: string, got: unknown): void { + if (got) { + pass++; + console.log(` \u2713 ${label}`); + } else { + fail++; + console.log(` \u2717 ${label} \u2014 falsy: ${JSON.stringify(got)}`); + } +} + +function throws(label: string, fn: () => unknown, re?: RegExp): void { + try { + fn(); + fail++; + console.log(` \u2717 ${label} \u2014 n'a pas lanc\u00e9 d'exception`); + } catch (e) { + const msg = String((e as Error)?.message ?? e); + if (!re || re.test(msg)) { + pass++; + console.log(` \u2713 ${label}`); + } else { + fail++; + console.log(` \u2717 ${label} \u2014 exception inattendue: ${msg}`); + } + } +} + +console.log("\n[1] Rep\u00e8res Qlik (skill qlik)"); +for (const a of QLIK_DATE_ANCHORS) { + eq(`isoDateToQlikSerial(${a.iso}) = ${a.serial}`, isoDateToQlikSerial(a.iso), a.serial); +} + +console.log("\n[2] Round-trip s\u00e9rial \u2194 ISO"); +for (const a of QLIK_DATE_ANCHORS) { + eq(`qlikSerialToIsoDate(${a.serial}) = ${a.iso}`, qlikSerialToIsoDate(a.serial), a.iso); +} + +console.log("\n[3] Fen\u00eatre grille (now=2026-06-21) \u2192 2025-06-01 .. 2026-05-31"); +const f = buildGridNetworkQlikDateFilter(new Date("2026-06-21T12:00:00Z")); +eq("dateDebut", f.dateDebut, "2025-06-01"); +eq("dateFin", f.dateFin, "2026-05-31"); +eq("qStart", f.qStart, 45809); +eq("qEnd", f.qEnd, 46173); +eq("label", f.label, "2025-06_2026-05"); +eq("setAnalysis", f.setAnalysis, `Date={">=45809<=46173"}`); +eq("dailySerials.length = 365", f.dailySerials.length, 365); +eq("dailySerials[0]", f.dailySerials[0], 45809); +eq("dailySerials[364]", f.dailySerials[364], 46173); + +console.log("\n[4] Fen\u00eatre grille (now=2026-03-15) \u2192 2025-03-01 .. 2026-02-28"); +const f2 = buildGridNetworkQlikDateFilter(new Date("2026-03-15T08:00:00Z")); +eq("dateDebut", f2.dateDebut, "2025-03-01"); +eq("dateFin", f2.dateFin, "2026-02-28"); +eq("qStart", f2.qStart, 45717); +eq("qEnd", f2.qEnd, 46081); + +console.log("\n[5] Fen\u00eatre grille (now=2025-01-05) \u2192 2024-01-01 .. 2024-12-31 (ann\u00e9e bissextile, 366 j)"); +const f3 = buildGridNetworkQlikDateFilter(new Date("2025-01-05T08:00:00Z")); +eq("dateDebut", f3.dateDebut, "2024-01-01"); +eq("dateFin", f3.dateFin, "2024-12-31"); +eq("dailySerials.length = 366", f3.dailySerials.length, 366); + +console.log("\n[6] chunkSerials"); +const chunks = chunkSerials(f.dailySerials, 100); +eq("chunks.length pour 365/100", chunks.length, 4); +eq("chunks[3].length = 65", chunks[3].length, 65); +eq("chunks[0][0]", chunks[0][0], 45809); +eq("chunks[3][64]", chunks[3][64], 46173); + +console.log("\n[7] Robustesse"); +throws("format invalide '2025/01/01'", () => isoDateToQlikSerial("2025/01/01"), /YYYY-MM-DD/); +throws("format vide", () => isoDateToQlikSerial(""), /YYYY-MM-DD/); +truthy("label non vide pour fen\u00eatre valide", f.label && f.label.length > 0); +truthy("setAnalysis commence par 'Date='", f.setAnalysis.startsWith("Date=")); + +console.log(`\n=== ${pass} pass\u00e9s, ${fail} \u00e9chou\u00e9s ===`); +if (fail > 0) process.exit(1); diff --git a/src/app/api/qlik/sync/route.ts b/src/app/api/qlik/sync/route.ts index a98da7f..3d833c6 100644 --- a/src/app/api/qlik/sync/route.ts +++ b/src/app/api/qlik/sync/route.ts @@ -3,6 +3,7 @@ import { auth } from "@/lib/auth"; import { fetchNetworkMetricsPlaywright } from "@/lib/qlik-playwright"; import { upsertNetworkMetrics } from "@/lib/qlik-network-cache"; import { pgGetArticlesByFournisseur } from "@/lib/pg-ff-client"; +import { buildGridNetworkQlikDateFilter } from "@/lib/qlik-date-range"; // Synchro potentiellement longue (extraction hypercube paginee). export const maxDuration = 300; @@ -45,8 +46,9 @@ export async function POST(req: NextRequest) { }); } - console.log(`[api/qlik/sync] → lancement extraction Qlik pour ${codes.length} codes…`); - const metrics = await fetchNetworkMetricsPlaywright(codes); + const dateFilter = buildGridNetworkQlikDateFilter(); + console.log(`[api/qlik/sync] → lancement extraction Qlik pour ${codes.length} codes — période ${dateFilter.label} (${dateFilter.dateDebut} → ${dateFilter.dateFin})…`); + const metrics = await fetchNetworkMetricsPlaywright(codes, undefined, dateFilter); console.log(`[api/qlik/sync] ← Qlik a renvoyé ${metrics.size} produits réseau`); const count = await upsertNetworkMetrics([...metrics.values()]); console.log(`[api/qlik/sync] ${count} lignes upsert dans le cache`); @@ -56,6 +58,9 @@ export async function POST(req: NextRequest) { requested: codes.length, fetched: metrics.size, upserted: count, + periode: dateFilter.label, + dateDebut: dateFilter.dateDebut, + dateFin: dateFilter.dateFin, fetchedAt: new Date().toISOString(), }); } catch (e) { diff --git a/src/lib/qlik-client.ts b/src/lib/qlik-client.ts index b96d30d..0feca91 100644 --- a/src/lib/qlik-client.ts +++ b/src/lib/qlik-client.ts @@ -21,6 +21,7 @@ import { WebSocket } from "ws"; import { randomBytes } from "node:crypto"; import { readFileSync } from "node:fs"; import { join } from "node:path"; +import { buildGridNetworkQlikDateFilter, chunkSerials, type QlikDateFilter } from "@/lib/qlik-date-range"; /** Lit les réglages Qlik depuis data/.db-config.json (configurés dans Paramètres). */ function readQlikFileConfig(): { qlikHost?: string; qlikUser?: string; qlikPassword?: string } { @@ -254,6 +255,7 @@ type Matrix = Array>; export async function fetchNetworkMetrics( codeCentraux?: string[], cfg = getQlikConfig(), + dateFilter: QlikDateFilter | null = buildGridNetworkQlikDateFilter(), ): Promise> { if (!cfg.appNetwork) throw new Error("[qlik] QLIK_APP_NETWORK manquant"); const wanted = codeCentraux ? new Set(codeCentraux) : null; @@ -265,6 +267,38 @@ export async function fetchNetworkMetrics( const openRes = await conn.rpc("OpenDoc", { qDocName: cfg.appNetwork }); const docHandle = (((openRes.qReturn as Record) ?? {}).qHandle as number) ?? 1; + // Sélection du champ `Date` sur la fenêtre 12 mois complets glissants + // (alignement avec la timeline de la grille). Équivalent à un set + // analysis `{=…<=…"}>}` appliqué aux master measures. + let dateFieldHandle: number | null = null; + if (dateFilter) { + const gf = await conn.rpc("GetField", { qFieldName: "Date" }, docHandle); + dateFieldHandle = (((gf.qReturn as Record) ?? {}).qHandle as number); + if (typeof dateFieldHandle !== "number") { + console.warn(`[qlik] champ Date introuvable, extraction sans filtre date`); + dateFilter = null; + dateFieldHandle = null; + } else { + const batches = chunkSerials(dateFilter.dailySerials, 500); + for (let i = 0; i < batches.length; i++) { + const batch = batches[i]; + const sel = await conn.rpc( + "SelectValues", + { + qFieldValues: batch.map((s) => ({ qNum: s, qText: String(s) })), + qToggleMode: false, + qSoftLock: true, + }, + dateFieldHandle, + ); + console.log(`[qlik] date filter lot ${i + 1}/${batches.length} (${batch.length} serials) — ${dateFilter.label}`); + // Si SelectValues renvoie une erreur qError, on continue quand même : + // l'absence de champ Date typé numérique lèverait une erreur Engine explicite. + void sel; + } + } + } + const pageHeight = 1400; // 6 colonnes * 1400 = 8400 cellules max const createRes = await conn.rpc("CreateSessionObject", { qProp: hyperCubeDef(cfg, pageHeight) }, docHandle); const objHandle = (((createRes.qReturn as Record) ?? {}).qHandle as number); @@ -291,6 +325,7 @@ export async function fetchNetworkMetrics( nbMagasinsReseau: Number(row[3]?.qNum ?? 0) || 0, caParMagasinReseau: Number(row[4]?.qNum ?? 0) || 0, margePctReseau: Number(row[5]?.qNum ?? 0) || 0, + periode: dateFilter?.label, }); } if (matrix.length < pageHeight) break; diff --git a/src/lib/qlik-date-range.ts b/src/lib/qlik-date-range.ts new file mode 100644 index 0000000..6ada20b --- /dev/null +++ b/src/lib/qlik-date-range.ts @@ -0,0 +1,137 @@ +/** + * CollectFlow — Helper de fenêtre temporelle pour Qlik réseau. + * + * Aligne la timeline d'extraction Qlik réseau sur la timeline de la grille : + * - 12 mois complets glissants + * - exclut systématiquement le mois courant + * + * Référence utilisée : `buildLast12MonthsRange()` de `api-ff-client.ts` + * (même source que la grille). On duplique la logique ici (sans dépendance + * Next.js) pour rester testable en CLI pur Node. + * + * Qlik référence les dates par un sérial numérique (jours écoulés depuis + * l'epoch Qlik 1899-12-30). Repères validés : + * 2025-01-01 → 45658 + * 2025-06-19 → 45827 + * 2026-01-01 → 46023 + * 2026-06-19 → 46192 + * + * Le filtre côté Qlik s'applique via une sélection de champ `Date` (skill qlik) : + * - valeurs numériques = serials Qlik + * - la sélection par défaut (`{}`) est héritée par toutes + * les master measures qui référencent le champ `Date` (cas de l'app + * "Magasins Vision Consolidée"). + * + * Pourquoi `SelectValues` sur le champ plutôt que set analysis dans `qDef` ? + * - les master items sont référencés par `qLibraryId` (réutilisation sans + * devoir dupliquer l'expression) + * - `qLibraryId` + `qDef` ensemble n'est pas supporté par le moteur QIX + * - sélectionner toutes les dates de la fenêtre via `SelectValues` revient + * fonctionnellement à `{=…<=…"}>}` pour les mesures qui touchent + * ce champ, sans dépendre de l'expression interne des master items + * + * Pas de secrets ici. Pure fonction utilitaire testable en CLI Node. + */ + +/** Epoch Qlik = 1899-12-30 (équivalent Excel/Lotus 1-2-3). */ +const QLIK_EPOCH_MS = Date.UTC(1899, 11, 30); // 1899-12-30T00:00:00Z + +/** Repères de validation (cf. skill qlik). */ +export const QLIK_DATE_ANCHORS: ReadonlyArray<{ iso: string; serial: number }> = [ + { iso: "2025-01-01", serial: 45658 }, + { iso: "2025-06-19", serial: 45827 }, + { iso: "2026-01-01", serial: 46023 }, + { iso: "2026-06-19", serial: 46192 }, +]; + +/** + * Convertit une date ISO `YYYY-MM-DD` en sérial Qlik numérique. + * Inverse documenté : Qlik epoch = 1899-12-30 (jour 0). + * + * @throws si l'entrée n'est pas au format `YYYY-MM-DD`. + */ +export function isoDateToQlikSerial(isoDate: string): number { + const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(isoDate); + if (!m) throw new Error(`isoDateToQlikSerial: format attendu YYYY-MM-DD, reçu "${isoDate}"`); + const y = Number(m[1]); + const mo = Number(m[2]); + const d = Number(m[3]); + const ms = Date.UTC(y, mo - 1, d); + const serial = Math.round((ms - QLIK_EPOCH_MS) / 86_400_000); + return serial; +} + +/** + * Convertit un sérial Qlik en date ISO `YYYY-MM-DD` (utile pour tests/diagnostics). + */ +export function qlikSerialToIsoDate(serial: number): string { + const ms = QLIK_EPOCH_MS + serial * 86_400_000; + const d = new Date(ms); + const y = d.getUTCFullYear(); + const mo = String(d.getUTCMonth() + 1).padStart(2, "0"); + const da = String(d.getUTCDate()).padStart(2, "0"); + return `${y}-${mo}-${da}`; +} + +export interface QlikDateFilter { + /** Date ISO du 1er jour de la fenêtre (inclus). */ + dateDebut: string; + /** Date ISO du dernier jour de la fenêtre (inclus). */ + dateFin: string; + /** Sérial Qlik du 1er jour (= `Date >= …`). */ + qStart: number; + /** Sérial Qlik du dernier jour (= `Date <= …`). */ + qEnd: number; + /** Libellé humain `YYYY-MM_YYYY-MM` pour logs / cache. */ + label: string; + /** Liste exhaustive des serials journaliers `[qStart..qEnd]` pour SelectValues. */ + dailySerials: number[]; + /** Expression set analysis prête à intégrer dans un `qDef` de mesure. */ + setAnalysis: string; +} + +/** + * Calcule la fenêtre 12 mois complets (excluant mois courant). + * Par défaut, utilise `now = new Date()`. Testable via injection de `now`. + * + * Logique alignée sur `buildLast12MonthsRange()` de `src/lib/api-ff-client.ts`. + * Si la source officielle change, mettre à jour ici aussi (garder en sync). + */ +export function buildGridNetworkQlikDateFilter(now: Date = new Date()): QlikDateFilter { + // Fin : dernier jour du mois précédent le mois courant + const endMonth = new Date(now.getFullYear(), now.getMonth(), 0); + // Début : 12 mois avant le mois courant (1er du mois) + const startMonth = new Date(now.getFullYear(), now.getMonth() - 12, 1); + + const dateDebut = startMonth.toISOString().slice(0, 10); + const dateFin = endMonth.toISOString().slice(0, 10); + + const qStart = isoDateToQlikSerial(dateDebut); + const qEnd = isoDateToQlikSerial(dateFin); + + if (qEnd < qStart) { + throw new Error( + `buildGridNetworkQlikDateFilter: fenêtre inversée qEnd=${qEnd} < qStart=${qStart} (dateDebut=${dateDebut} dateFin=${dateFin})`, + ); + } + + const dailySerials: number[] = []; + for (let s = qStart; s <= qEnd; s++) dailySerials.push(s); + + const label = `${dateDebut.slice(0, 7)}_${dateFin.slice(0, 7)}`; + const setAnalysis = `Date={">=${qStart}<=${qEnd}"}`; + + return { dateDebut, dateFin, qStart, qEnd, label, dailySerials, setAnalysis }; +} + +/** + * Découpe les serials en lots pour appels `SelectValues` paginés. + * Par défaut 500 valeurs/appel (largement < limite Qlik ~10k par requête). + */ +export function chunkSerials(serials: number[], batchSize = 500): number[][] { + const out: number[][] = []; + for (let i = 0; i < serials.length; i += batchSize) { + out.push(serials.slice(i, i + batchSize)); + } + return out; +} diff --git a/src/lib/qlik-playwright.ts b/src/lib/qlik-playwright.ts index d5fff3c..158f305 100644 --- a/src/lib/qlik-playwright.ts +++ b/src/lib/qlik-playwright.ts @@ -14,6 +14,7 @@ import "server-only"; import { chromium, type Browser } from "playwright-core"; import { getQlikConfig, qlikNtlmSession, type QlikConfig, type NetworkMetric } from "@/lib/qlik-client"; +import { buildGridNetworkQlikDateFilter, chunkSerials, type QlikDateFilter } from "@/lib/qlik-date-range"; let browserPromise: Promise | null = null; @@ -35,11 +36,15 @@ interface InPageResult { ok: boolean; rows?: Array>; size export async function fetchNetworkMetricsPlaywright( codeCentraux?: string[], cfg: QlikConfig = getQlikConfig(), + dateFilter: QlikDateFilter | null = buildGridNetworkQlikDateFilter(), ): Promise> { if (!cfg.appNetwork) throw new Error("[qlik-pw] QLIK_APP_NETWORK manquant"); if (!cfg.user || !cfg.password) throw new Error("[qlik-pw] identifiants Qlik manquants"); console.log(`[qlik-pw] host=${cfg.host} app=${cfg.appNetwork} — ${(codeCentraux ?? []).length} codes à sélectionner`); + if (dateFilter) { + console.log(`[qlik-pw] filtre Date=${dateFilter.label} (${dateFilter.dateDebut} → ${dateFilter.dateFin}, ${dateFilter.setAnalysis})`); + } const sess = await qlikNtlmSession(cfg); console.log(`[qlik-pw] session NTLM OK, cookie=${sess.cookie.split("=")[0]}=…`); const [cookieName, ...rest] = sess.cookie.split("="); @@ -75,7 +80,7 @@ export async function fetchNetworkMetricsPlaywright( console.log(`[qlik-pw] csrf-token capturé, ouverture du websocket Engine in-page…`); const result = (await page.evaluate( - ({ app, dim, mca, mqte, mnb, mcamag, mmarge, codes, token }: { app: string; dim: string; mca: string; mqte: string; mnb: string; mcamag: string; mmarge: string; codes: string[]; token: string }) => + ({ app, dim, mca, mqte, mnb, mcamag, mmarge, codes, token, dateBatches, dateLabel }: { app: string; dim: string; mca: string; mqte: string; mnb: string; mcamag: string; mmarge: string; codes: string[]; token: string; dateBatches: number[][]; dateLabel: string | null }) => new Promise((resolve) => { const loc = (window as unknown as { location: Location }).location; const url = `wss://${loc.host}/app/${app}?reloadUri=${encodeURIComponent(loc.href)}&qlik-csrf-token=${token}`; @@ -103,6 +108,28 @@ export async function fetchNetworkMetricsPlaywright( const open = await rpc("OpenDoc", { qDocName: app }); const doc = (open.qReturn as { qHandle: number }).qHandle; console.log("OpenDoc OK handle=" + doc); + + // Aligne Qlik sur la timeline grille : 12 mois complets glissants, + // hors mois courant. La sélection du champ Date est héritée par les + // master measures du cube, sans dupliquer leurs expressions. + if (dateBatches.length) { + try { + const df = await rpc("GetField", { qFieldName: "Date" }, doc); + const dfh = (df.qReturn as { qHandle: number }).qHandle; + for (let i = 0; i < dateBatches.length; i++) { + const batch = dateBatches[i]; + await rpc( + "SelectValues", + { qFieldValues: batch.map((s) => ({ qNum: s, qText: String(s) })), qToggleMode: false, qSoftLock: true }, + dfh, + ); + console.log("filtre Date " + (i + 1) + "/" + dateBatches.length + " — " + batch.length + " jours — " + dateLabel); + } + } catch (e) { + console.log("filtre Date ignoré: " + String((e as Error)?.message || e)); + } + } + // Handle du champ de sélection (réutilisé pour chaque lot). let fh = -1; if (codes.length) { @@ -156,7 +183,19 @@ export async function fetchNetworkMetricsPlaywright( } catch (e) { try { ws.close(); } catch { /* noop */ } fail(String((e as Error)?.message || e)); } })(); }), - { app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId, mcamag: cfg.measCaMagId, mmarge: cfg.measMargePctId, codes: codeCentraux ?? [], token: csrfToken }, + { + app: cfg.appNetwork, + dim: cfg.dimCodeArticleId, + mca: cfg.measCaId, + mqte: cfg.measQteId, + mnb: cfg.measNbMagId, + mcamag: cfg.measCaMagId, + mmarge: cfg.measMargePctId, + codes: codeCentraux ?? [], + token: csrfToken, + dateBatches: dateFilter ? chunkSerials(dateFilter.dailySerials, 500) : [], + dateLabel: dateFilter?.label ?? null, + }, )) as InPageResult; if (!result.ok) throw new Error(`[qlik-pw] ${result.error}`); @@ -167,7 +206,7 @@ export async function fetchNetworkMetricsPlaywright( const code = String(r[0]).trim(); if (!code || code === "-") continue; if (wanted && !wanted.has(code)) continue; - out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0, caParMagasinReseau: Number(r[4]) || 0, margePctReseau: Number(r[5]) || 0 }); + out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0, caParMagasinReseau: Number(r[4]) || 0, margePctReseau: Number(r[5]) || 0, periode: dateFilter?.label }); } console.log(`[qlik-pw] ${out.size} produits réseau (cube ${result.size})`); // Échantillon brut pour vérifier que les 4 nouvelles mesures (cols 4-7) renvoient des valeurs