fix(qlik): batch SelectValues to avoid Engine abort + drop couv stock metric

Large supplier (4816 codes) triggered Qlik Engine "Request aborted"
(code 15): a single SelectValues over all codes forced the hypercube to
calculate too many rows at once. Now select in 1200-code batches and
accumulate, keeping each calc small. 1417 codes worked before, 4816 did
not - chunking scales to any supplier.

Also remove the "Couverture de stock" network column end-to-end (grid,
types, Qlik clients, cache, schema, db-init), shifting the marge measure
index from 6 to 5 and the hypercube width from 7 to 6.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
MichaelandClaude Opus 4.8 committed 2026-06-21 10:55:03 +02:00
1 parent 8cc95cb387
commit de8a435c14
8 files changed
+39 -60

No files matched your search

+1 -2
View File
@@ -103,7 +103,6 @@ async function main() {
"qte_reseau" numeric(14, 2),
"nb_magasins_reseau" integer,
"ca_par_magasin_reseau" numeric(16, 2),
"couverture_stock_reseau" numeric(12, 3),
"marge_pct_reseau" numeric(8, 4),
"periode" varchar(20),
"fetched_at" timestamp DEFAULT now()
@@ -113,9 +112,9 @@ async function main() {
await tempPool.query(`
ALTER TABLE "qlik_network_metrics"
ADD COLUMN IF NOT EXISTS "ca_par_magasin_reseau" numeric(16, 2),
ADD COLUMN IF NOT EXISTS "couverture_stock_reseau" numeric(12, 3),
ADD COLUMN IF NOT EXISTS "marge_pct_reseau" numeric(8, 4);
ALTER TABLE "qlik_network_metrics" DROP COLUMN IF EXISTS "rupture_pct_reseau";
ALTER TABLE "qlik_network_metrics" DROP COLUMN IF EXISTS "couverture_stock_reseau";
`);
console.log("[DB Init] Table qlik_network_metrics is verified/created.");
-2
View File
@@ -119,8 +119,6 @@ export const qlikNetworkMetrics = pgTable("qlik_network_metrics", {
nbMagasinsReseau: integer("nb_magasins_reseau"),
/** CA moyen par magasin (productivité normalisée présence) */
caParMagasinReseau: numeric("ca_par_magasin_reseau", { precision: 16, scale: 2 }),
/** Couverture de stock en quantité (rotation) */
couvertureStockReseau: numeric("couverture_stock_reseau", { precision: 12, scale: 3 }),
/** Taux de marge réseau (ratio brut Qlik) */
margePctReseau: numeric("marge_pct_reseau", { precision: 8, scale: 4 }),
/** Période couverte (libre, ex "12m" ou "2025") */
@@ -484,7 +484,6 @@ async function enrichWithNetworkMetrics(productMap: Map<string, ProductRow>): Pr
product.qteReseau = m.qteReseau;
product.nbMagasinsReseau = m.nbMagasinsReseau;
product.caParMagasinReseau = m.caParMagasinReseau;
product.couvertureStockReseau = m.couvertureStockReseau;
product.margePctReseau = m.margePctReseau;
product.tauxPresenceReseau = m.nbMagasinsReseau / NB_MAGASINS_RESEAU;
product.networkFetchedAt = m.fetchedAt ?? undefined;
+2 -18
View File
@@ -178,7 +178,7 @@ const GridRow = React.memo(({ virtualRow, row, rowHeight, isSelected, columnVisi
>
{row.getVisibleCells().map((cell: Cell<ProductRow, unknown>) => {
const isFlexible = cell.column.id === "libelle1" || cell.column.id === "libelle3";
const isCenter = cell.column.id === "totalQuantite" || cell.column.id === "totalCa" || cell.column.id === "totalMarge" || cell.column.id.startsWith("month_") || cell.column.id === "gammeInitial" || cell.column.id === "caReseau" || cell.column.id === "qteReseau" || cell.column.id === "nbMagasinsReseau" || cell.column.id === "caParMagasinReseau" || cell.column.id === "couvertureStockReseau" || cell.column.id === "margePctReseau" || cell.column.id === "tauxPresenceReseau" || cell.column.id === "gamme";
const isCenter = cell.column.id === "totalQuantite" || cell.column.id === "totalCa" || cell.column.id === "totalMarge" || cell.column.id.startsWith("month_") || cell.column.id === "gammeInitial" || cell.column.id === "caReseau" || cell.column.id === "qteReseau" || cell.column.id === "nbMagasinsReseau" || cell.column.id === "caParMagasinReseau" || cell.column.id === "margePctReseau" || cell.column.id === "tauxPresenceReseau" || cell.column.id === "gamme";
const size = cell.column.getSize();
return (
<td
@@ -573,22 +573,6 @@ export function HeatmapGrid({ onSelectionChange, isAdmin }: HeatmapGridProps) {
);
},
},
{
accessorKey: "couvertureStockReseau",
header: () => <div className="text-center w-full">Couv.<br/><span className="text-[9px] opacity-60">Stock</span></div>,
size: 70,
cell: ({ getValue }) => {
const val = getValue<number | undefined>();
if (val == null) return <div className="text-center text-[12px]" style={{ color: "var(--text-secondary)" }}>-</div>;
// Couverture basse = rotation rapide = bon signal d'achat
const color = val <= 2 ? "text-emerald-500" : val <= 4 ? "text-amber-500" : "text-rose-500";
return (
<div className={cn("text-center font-bold text-[12px] tabular-nums", color)}>
{val.toLocaleString("fr-FR", { maximumFractionDigits: 1 })}
</div>
);
},
},
{
accessorKey: "margePctReseau",
header: () => <div className="text-center w-full">Marge %<br/><span className="text-[9px] opacity-60">Réseau</span></div>,
@@ -823,7 +807,7 @@ export function HeatmapGrid({ onSelectionChange, isAdmin }: HeatmapGridProps) {
<tr key={headerGroup.id} className="flex w-full">
{headerGroup.headers.map((header) => {
const isFlexible = header.column.id === "libelle1" || header.column.id === "libelle3";
const isCenter = header.column.id === "totalQuantite" || header.column.id === "totalCa" || header.column.id === "totalMarge" || header.column.id.startsWith("month_") || header.column.id === "gammeInitial" || header.column.id === "caReseau" || header.column.id === "qteReseau" || header.column.id === "nbMagasinsReseau" || header.column.id === "caParMagasinReseau" || header.column.id === "couvertureStockReseau" || header.column.id === "margePctReseau" || header.column.id === "tauxPresenceReseau" || header.column.id === "gamme";
const isCenter = header.column.id === "totalQuantite" || header.column.id === "totalCa" || header.column.id === "totalMarge" || header.column.id.startsWith("month_") || header.column.id === "gammeInitial" || header.column.id === "caReseau" || header.column.id === "qteReseau" || header.column.id === "nbMagasinsReseau" || header.column.id === "caParMagasinReseau" || header.column.id === "margePctReseau" || header.column.id === "tauxPresenceReseau" || header.column.id === "gamme";
const size = header.getSize();
return (
<th
+4 -10
View File
@@ -49,7 +49,6 @@ export interface QlikConfig {
measQteId: string; // master measure "Quantite N"
measNbMagId: string; // master measure "Magasin Ventes Nb N"
measCaMagId: string; // master measure "CA par Magasin N"
measCouvQteId: string; // master measure "Couverture de stock Quantite N"
measMargePctId: string; // master measure "Marge % N"
tlsInsecure: boolean;
timeoutMs: number;
@@ -73,7 +72,6 @@ export function getQlikConfig(): QlikConfig {
// Master GUIDs de l'app "Magasins Vision Consolidée" (9872ee6e). Si l'engine
// les refuse (qId court attendu), override via env après discovery.
measCaMagId: process.env.QLIK_MEAS_CAMAG_ID ?? "16778c00-a4e0-41c7-ab1c-179810d78172",
measCouvQteId: process.env.QLIK_MEAS_COUVQTE_ID ?? "dbdf957d-a380-495f-bdc4-3f7b6688f3b5",
measMargePctId: process.env.QLIK_MEAS_MARGEPCT_ID ?? "ea1e8feb-49ba-4eb3-8db6-72dcfb4c1222",
tlsInsecure: (process.env.QLIK_TLS_INSECURE ?? "true") === "true",
timeoutMs: Number(process.env.QLIK_TIMEOUT_MS ?? "60000"),
@@ -87,8 +85,6 @@ export interface NetworkMetric {
nbMagasinsReseau: number;
/** CA moyen par magasin (productivité normalisée présence) */
caParMagasinReseau: number;
/** Couverture de stock en quantité (rotation : bas = tourne vite) */
couvertureStockReseau: number;
/** Taux de marge réseau (ratio brut Qlik, ex 0.32 = 32%) */
margePctReseau: number;
periode?: string;
@@ -244,10 +240,9 @@ function hyperCubeDef(cfg: QlikConfig, height: number) {
{ qLibraryId: cfg.measQteId },
{ qLibraryId: cfg.measNbMagId },
{ qLibraryId: cfg.measCaMagId },
{ qLibraryId: cfg.measCouvQteId },
{ qLibraryId: cfg.measMargePctId },
],
qInitialDataFetch: [{ qTop: 0, qLeft: 0, qWidth: 7, qHeight: height }],
qInitialDataFetch: [{ qTop: 0, qLeft: 0, qWidth: 6, qHeight: height }],
qSuppressZero: false,
qSuppressMissing: true,
},
@@ -270,7 +265,7 @@ export async function fetchNetworkMetrics(
const openRes = await conn.rpc("OpenDoc", { qDocName: cfg.appNetwork });
const docHandle = (((openRes.qReturn as Record<string, unknown>) ?? {}).qHandle as number) ?? 1;
const pageHeight = 1400; // 7 colonnes * 1400 = 9800 cellules max
const pageHeight = 1400; // 6 colonnes * 1400 = 8400 cellules max
const createRes = await conn.rpc("CreateSessionObject", { qProp: hyperCubeDef(cfg, pageHeight) }, docHandle);
const objHandle = (((createRes.qReturn as Record<string, unknown>) ?? {}).qHandle as number);
@@ -278,7 +273,7 @@ export async function fetchNetworkMetrics(
for (;;) {
const dataRes = await conn.rpc(
"GetHyperCubeData",
{ qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 7, qHeight: pageHeight }] },
{ qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 6, qHeight: pageHeight }] },
objHandle,
);
const pages = (dataRes.qDataPages as Array<{ qMatrix?: Matrix }>) ?? [];
@@ -295,8 +290,7 @@ export async function fetchNetworkMetrics(
qteReseau: Number(row[2]?.qNum ?? 0) || 0,
nbMagasinsReseau: Number(row[3]?.qNum ?? 0) || 0,
caParMagasinReseau: Number(row[4]?.qNum ?? 0) || 0,
couvertureStockReseau: Number(row[5]?.qNum ?? 0) || 0,
margePctReseau: Number(row[6]?.qNum ?? 0) || 0,
margePctReseau: Number(row[5]?.qNum ?? 0) || 0,
});
}
if (matrix.length < pageHeight) break;
-4
View File
@@ -16,7 +16,6 @@ export interface NetworkMetricCached {
qteReseau: number;
nbMagasinsReseau: number;
caParMagasinReseau: number;
couvertureStockReseau: number;
margePctReseau: number;
fetchedAt: string | null;
}
@@ -40,7 +39,6 @@ export async function getNetworkMetricsByCodeCentrale(
qteReseau: Number(r.qteReseau ?? 0) || 0,
nbMagasinsReseau: Number(r.nbMagasinsReseau ?? 0) || 0,
caParMagasinReseau: Number(r.caParMagasinReseau ?? 0) || 0,
couvertureStockReseau: Number(r.couvertureStockReseau ?? 0) || 0,
margePctReseau: Number(r.margePctReseau ?? 0) || 0,
fetchedAt: r.fetchedAt ? new Date(r.fetchedAt).toISOString() : null,
});
@@ -58,7 +56,6 @@ export async function upsertNetworkMetrics(metrics: NetworkMetric[]): Promise<nu
qteReseau: String(m.qteReseau),
nbMagasinsReseau: m.nbMagasinsReseau,
caParMagasinReseau: String(m.caParMagasinReseau),
couvertureStockReseau: String(m.couvertureStockReseau),
margePctReseau: String(m.margePctReseau),
periode: m.periode ?? null,
fetchedAt: now,
@@ -79,7 +76,6 @@ export async function upsertNetworkMetrics(metrics: NetworkMetric[]): Promise<nu
qteReseau: sql`excluded.qte_reseau`,
nbMagasinsReseau: sql`excluded.nb_magasins_reseau`,
caParMagasinReseau: sql`excluded.ca_par_magasin_reseau`,
couvertureStockReseau: sql`excluded.couverture_stock_reseau`,
margePctReseau: sql`excluded.marge_pct_reseau`,
periode: sql`excluded.periode`,
fetchedAt: sql`excluded.fetched_at`,
+32 -21
View File
@@ -75,7 +75,7 @@ export async function fetchNetworkMetricsPlaywright(
console.log(`[qlik-pw] csrf-token capturé, ouverture du websocket Engine in-page…`);
const result = (await page.evaluate(
({ app, dim, mca, mqte, mnb, mcamag, mcouv, mmarge, codes, token }: { app: string; dim: string; mca: string; mqte: string; mnb: string; mcamag: string; mcouv: string; mmarge: string; codes: string[]; token: string }) =>
({ app, dim, mca, mqte, mnb, mcamag, mmarge, codes, token }: { app: string; dim: string; mca: string; mqte: string; mnb: string; mcamag: string; mmarge: string; codes: string[]; token: string }) =>
new Promise<InPageResult>((resolve) => {
const loc = (window as unknown as { location: Location }).location;
const url = `wss://${loc.host}/app/${app}?reloadUri=${encodeURIComponent(loc.href)}&qlik-csrf-token=${token}`;
@@ -103,12 +103,11 @@ export async function fetchNetworkMetricsPlaywright(
const open = await rpc("OpenDoc", { qDocName: app });
const doc = (open.qReturn as { qHandle: number }).qHandle;
console.log("OpenDoc OK handle=" + doc);
// Sélection des codes du fournisseur (sinon ~1,2M lignes)
// Handle du champ de sélection (réutilisé pour chaque lot).
let fh = -1;
if (codes.length) {
const gf = await rpc("GetField", { qFieldName: "Article Code" }, doc);
const fh = (gf.qReturn as { qHandle: number }).qHandle;
const sel = await rpc("SelectValues", { qFieldValues: codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh);
console.log("SelectValues " + codes.length + " codes → " + JSON.stringify(sel.qReturn));
fh = (gf.qReturn as { qHandle: number }).qHandle;
}
// Résoudre les master measures par TITRE (le qLibraryId fiable = qInfo.qId, ≠ GUID QRS).
const norm = (s: string) => s.replace(/\s+/g, " ").trim().toLowerCase();
@@ -119,33 +118,45 @@ export async function fetchNetworkMetricsPlaywright(
for (const it of items) { const t = it.qMeta?.title ?? it.qData?.title ?? ""; if (t) byTitle.set(norm(t), it.qInfo.qId); }
const pickMeasure = (title: string, fallback: string) => byTitle.get(norm(title)) ?? fallback;
const rCamag = pickMeasure("CA par Magasin N", mcamag);
const rCouv = pickMeasure("Couverture de stock Quantité N", mcouv);
const rMarge = pickMeasure("Marge % N", mmarge);
console.log("measures résolues par titre: " + JSON.stringify({ rCamag, rCouv, rMarge }) + " (sur " + items.length + " mesures)");
console.log("measures résolues par titre: " + JSON.stringify({ rCamag, rMarge }) + " (sur " + items.length + " mesures)");
const obj = await rpc("CreateSessionObject", { qProp: { qInfo: { qType: "cf-net" }, qHyperCubeDef: {
qDimensions: [{ qLibraryId: dim }],
qMeasures: [{ qLibraryId: mca }, { qLibraryId: mqte }, { qLibraryId: mnb }, { qLibraryId: rCamag }, { qLibraryId: rCouv }, { qLibraryId: rMarge }],
qMeasures: [{ qLibraryId: mca }, { qLibraryId: mqte }, { qLibraryId: mnb }, { qLibraryId: rCamag }, { qLibraryId: rMarge }],
qInitialDataFetch: [],
} } }, doc);
const oh = (obj.qReturn as { qHandle: number }).qHandle;
const layout = await rpc("GetLayout", {}, oh);
const size = ((layout.qLayout as { qHyperCube: { qSize: { qcy: number } } }).qHyperCube).qSize.qcy;
console.log("cube qSize.qcy=" + size + " lignes");
const out: Array<Array<string | number>> = [];
const PAGE = 1400;
for (let top = 0; top < size; top += PAGE) {
const d = await rpc("GetHyperCubeData", { qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 7, qHeight: PAGE }] }, oh);
const matrix = (d.qDataPages as Array<{ qMatrix?: Array<Array<{ qText?: string; qNum?: number }>> }>)?.[0]?.qMatrix ?? [];
if (!matrix.length) break;
for (const r of matrix) out.push([String(r[0]?.qText ?? ""), Number(r[1]?.qNum) || 0, Number(r[2]?.qNum) || 0, Number(r[3]?.qNum) || 0, Number(r[4]?.qNum) || 0, Number(r[5]?.qNum) || 0, Number(r[6]?.qNum) || 0]);
if (matrix.length < PAGE) break;
// Lots de sélection : une grande sélection (>~1500 codes) fait calculer le cube
// sur trop de lignes d'un coup → l'Engine annule (code 15 "Request aborted").
// On sélectionne par paquets et on accumule.
const BATCH = 1200;
const batches: Array<string[] | null> = codes.length
? Array.from({ length: Math.ceil(codes.length / BATCH) }, (_, i) => codes.slice(i * BATCH, i * BATCH + BATCH))
: [null];
for (let b = 0; b < batches.length; b++) {
const batch = batches[b];
if (batch) {
const sel = await rpc("SelectValues", { qFieldValues: batch.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh);
console.log("lot " + (b + 1) + "/" + batches.length + " — SelectValues " + batch.length + " codes → " + JSON.stringify(sel.qReturn));
}
const layout = await rpc("GetLayout", {}, oh);
const size = ((layout.qLayout as { qHyperCube: { qSize: { qcy: number } } }).qHyperCube).qSize.qcy;
for (let top = 0; top < size; top += PAGE) {
const d = await rpc("GetHyperCubeData", { qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 6, qHeight: PAGE }] }, oh);
const matrix = (d.qDataPages as Array<{ qMatrix?: Array<Array<{ qText?: string; qNum?: number }>> }>)?.[0]?.qMatrix ?? [];
if (!matrix.length) break;
for (const r of matrix) out.push([String(r[0]?.qText ?? ""), Number(r[1]?.qNum) || 0, Number(r[2]?.qNum) || 0, Number(r[3]?.qNum) || 0, Number(r[4]?.qNum) || 0, Number(r[5]?.qNum) || 0]);
if (matrix.length < PAGE) break;
}
}
ws.close();
resolve({ ok: true, rows: out, size });
resolve({ ok: true, rows: out, size: out.length });
} catch (e) { try { ws.close(); } catch { /* noop */ } fail(String((e as Error)?.message || e)); }
})();
}),
{ app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId, mcamag: cfg.measCaMagId, mcouv: cfg.measCouvQteId, mmarge: cfg.measMargePctId, codes: codeCentraux ?? [], token: csrfToken },
{ app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId, mcamag: cfg.measCaMagId, mmarge: cfg.measMargePctId, codes: codeCentraux ?? [], token: csrfToken },
)) as InPageResult;
if (!result.ok) throw new Error(`[qlik-pw] ${result.error}`);
@@ -156,12 +167,12 @@ export async function fetchNetworkMetricsPlaywright(
const code = String(r[0]).trim();
if (!code || code === "-") continue;
if (wanted && !wanted.has(code)) continue;
out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0, caParMagasinReseau: Number(r[4]) || 0, couvertureStockReseau: Number(r[5]) || 0, margePctReseau: Number(r[6]) || 0 });
out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0, caParMagasinReseau: Number(r[4]) || 0, margePctReseau: Number(r[5]) || 0 });
}
console.log(`[qlik-pw] ${out.size} produits réseau (cube ${result.size})`);
// Échantillon brut pour vérifier que les 4 nouvelles mesures (cols 4-7) renvoient des valeurs
const sample = (result.rows ?? []).slice(0, 3).map((r) => ({
code: r[0], ca: r[1], qte: r[2], nbMag: r[3], caMag: r[4], couv: r[5], margePct: r[6],
code: r[0], ca: r[1], qte: r[2], nbMag: r[3], caMag: r[4], margePct: r[5],
}));
console.log(`[qlik-pw] échantillon lignes:`, JSON.stringify(sample));
return out;
-2
View File
@@ -115,8 +115,6 @@ export interface ProductRow {
nbMagasinsReseau?: number;
/** CA moyen par magasin réseau (productivité normalisée présence) */
caParMagasinReseau?: number;
/** Couverture de stock quantité réseau (rotation : bas = tourne vite) */
couvertureStockReseau?: number;
/** Taux de marge réseau (ratio brut Qlik, ex 0.32 = 32%) */
margePctReseau?: number;
/** Taux de présence réseau = nbMagasinsReseau / 270 */