mirror of
https://github.com/R0m1k3/CollectFlow.git
synced 2026-10-20 21:48:33 +02:00
perf(qlik): optimize network sync extraction
This commit is contained in:
R0m1k3 committed
1 parent
1cb6c2496d
commit
abac5c5ee5
5 files changed
+468
-80
No files matched your search
+184
-60
@@ -31,7 +31,47 @@ async function getBrowser(): Promise<Browser> {
|
||||
return browserPromise;
|
||||
}
|
||||
|
||||
interface InPageResult { ok: boolean; rows?: Array<Array<string | number>>; size?: number; error?: string; diag?: Record<string, unknown>; }
|
||||
interface InPageResult {
|
||||
ok: boolean;
|
||||
rows?: Array<Array<string | number>>;
|
||||
size?: number;
|
||||
error?: string;
|
||||
diag?: Record<string, unknown>;
|
||||
// Compteurs agrégés pour le résumé timing final côté Node (cf. qlik-playwright).
|
||||
months?: number;
|
||||
batches?: number;
|
||||
code15Retries?: number;
|
||||
fallbackCount?: number;
|
||||
}
|
||||
|
||||
function envNumber(name: string, defaultValue: number, min: number, max: number): number {
|
||||
const raw = process.env[name];
|
||||
if (raw == null || raw.trim() === "") return defaultValue;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed)) return defaultValue;
|
||||
return Math.min(max, Math.max(min, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
function envNumberCsv(name: string, min: number, max: number): number[] {
|
||||
const raw = process.env[name];
|
||||
if (!raw) return [];
|
||||
return raw
|
||||
.split(",")
|
||||
.map((part) => Number(part.trim()))
|
||||
.filter((value) => Number.isFinite(value))
|
||||
.map((value) => Math.min(max, Math.max(min, Math.trunc(value))));
|
||||
}
|
||||
|
||||
function uniqueOrdered(values: number[]): number[] {
|
||||
const seen = new Set<number>();
|
||||
const out: number[] = [];
|
||||
for (const value of values) {
|
||||
if (seen.has(value)) continue;
|
||||
seen.add(value);
|
||||
out.push(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function fetchNetworkMetricsPlaywright(
|
||||
codeCentraux?: string[],
|
||||
@@ -41,6 +81,8 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
if (!cfg.appNetwork) throw new Error("[qlik-pw] QLIK_APP_NETWORK manquant");
|
||||
if (!cfg.user || !cfg.password) throw new Error("[qlik-pw] identifiants Qlik manquants");
|
||||
|
||||
// Démarre l'horloge pour le résumé timing final (cf. bloc [qlik-pw][summary]).
|
||||
const totalStart = Date.now();
|
||||
console.log(`[qlik-pw] host=${cfg.host} app=${cfg.appNetwork} — ${(codeCentraux ?? []).length} codes à sélectionner`);
|
||||
if (dateFilter) {
|
||||
console.log(`[qlik-pw] filtre Date=${dateFilter.label} (${dateFilter.dateDebut} → ${dateFilter.dateFin}, ${dateFilter.setAnalysis})`);
|
||||
@@ -49,6 +91,12 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
console.log(`[qlik-pw] session NTLM OK, cookie=${sess.cookie.split("=")[0]}=…`);
|
||||
const [cookieName, ...rest] = sess.cookie.split("=");
|
||||
const cookieValue = rest.join("=");
|
||||
const qlikSettleMs = envNumber("QLIK_SETTLE_MS", 150, 0, 2000);
|
||||
const qlikCode15MaxAttempts = envNumber("QLIK_CODE15_MAX_ATTEMPTS", 4, 1, 8);
|
||||
const qlikArticleBatchSize = envNumber("QLIK_ARTICLE_BATCH_SIZE", 150, 1, 500);
|
||||
const csvFallbacks = envNumberCsv("QLIK_ARTICLE_BATCH_FALLBACKS", 1, 500);
|
||||
const qlikArticleBatchFallbacks = uniqueOrdered([qlikArticleBatchSize, ...(csvFallbacks.length ? csvFallbacks : [75, 50]), 75, 50]);
|
||||
console.log(`[qlik-pw] tuning settle=${qlikSettleMs}ms code15Attempts=${qlikCode15MaxAttempts} batchFallbacks=${qlikArticleBatchFallbacks.join(">")}`);
|
||||
|
||||
const browser = await getBrowser();
|
||||
console.log(`[qlik-pw] chromium lancé`);
|
||||
@@ -89,11 +137,14 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
}));
|
||||
|
||||
const result = (await page.evaluate(
|
||||
({ app, dim, mca, mqte, mnb, mcamag, mmarge, codes, token, monthlyPayload, noDateMode }: {
|
||||
({ app, dim, mca, mqte, mnb, mcamag, mmarge, codes, token, monthlyPayload, noDateMode, qlikSettleMs, qlikCode15MaxAttempts, qlikArticleBatchFallbacks }: {
|
||||
app: string; dim: string; mca: string; mqte: string; mnb: string; mcamag: string; mmarge: string;
|
||||
codes: string[]; token: string;
|
||||
monthlyPayload: Array<{ label: string; dateDebut: string; dateFin: string; serials: number[] }>;
|
||||
noDateMode: boolean;
|
||||
qlikSettleMs: number;
|
||||
qlikCode15MaxAttempts: number;
|
||||
qlikArticleBatchFallbacks: number[];
|
||||
}) =>
|
||||
new Promise<InPageResult>((resolve) => {
|
||||
const loc = (window as unknown as { location: Location }).location;
|
||||
@@ -110,7 +161,12 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
ws.onmessage = (ev: MessageEvent) => {
|
||||
const m = JSON.parse(ev.data as string);
|
||||
if (m.method === "OnConnected") return res();
|
||||
if (m.id && pend.has(m.id)) { const x = pend.get(m.id)!; pend.delete(m.id); m.error ? x.rej(new Error(JSON.stringify(m.error))) : x.res(m.result); }
|
||||
if (m.id && pend.has(m.id)) {
|
||||
const x = pend.get(m.id)!;
|
||||
pend.delete(m.id);
|
||||
if (m.error) x.rej(new Error(JSON.stringify(m.error)));
|
||||
else x.res(m.result);
|
||||
}
|
||||
};
|
||||
ws.onerror = () => rej(new Error("ws error (403 ?)"));
|
||||
setTimeout(() => rej(new Error("ws timeout")), 30000);
|
||||
@@ -124,22 +180,34 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
return /"code"\s*:\s*15\b/.test(msg);
|
||||
};
|
||||
const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
|
||||
const nowMs = () => (typeof performance !== "undefined" ? performance.now() : Date.now());
|
||||
const timed = async <T>(etape: string, method: string, fn: () => Promise<T>, timings?: Record<string, number>): Promise<{ value: T; ms: number }> => {
|
||||
const start = nowMs();
|
||||
const value = await fn();
|
||||
const ms = Math.round(nowMs() - start);
|
||||
const label = `${etape}:${method}`;
|
||||
if (timings) timings[label] = (timings[label] ?? 0) + ms;
|
||||
console.log(`[qlik-pw][timing] ${label} ms=${ms}`);
|
||||
return { value, ms };
|
||||
};
|
||||
const timingSummary = (timings: Record<string, number>) => JSON.stringify(timings);
|
||||
const rpcWithRetry = async (
|
||||
etape: string,
|
||||
method: string,
|
||||
params: unknown,
|
||||
handle: number,
|
||||
maxAttempts = 4,
|
||||
onCode15Retry?: () => void,
|
||||
): Promise<{ [k: string]: unknown }> => {
|
||||
let lastErr: unknown;
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
for (let attempt = 1; attempt <= qlikCode15MaxAttempts; attempt++) {
|
||||
try {
|
||||
return await rpc(method, params, handle);
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
if (!isEngineCode15(e) || attempt === maxAttempts) throw e;
|
||||
const backoff = 150 * attempt;
|
||||
console.log(`[qlik-pw] retry code15 étape=${etape} méthode=${method} tentative=${attempt}/${maxAttempts} backoff=${backoff}ms`);
|
||||
if (!isEngineCode15(e) || attempt === qlikCode15MaxAttempts) throw e;
|
||||
onCode15Retry?.();
|
||||
const backoff = qlikSettleMs * attempt;
|
||||
console.log(`[qlik-pw] retry code15 étape=${etape} méthode=${method} tentative=${attempt}/${qlikCode15MaxAttempts} backoff=${backoff}ms`);
|
||||
await sleep(backoff);
|
||||
}
|
||||
}
|
||||
@@ -180,6 +248,11 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
// causé par la réutilisation du même objet sur des centaines de recalculs.
|
||||
|
||||
const out: Array<Array<string | number>> = [];
|
||||
// Compteurs agrégés pour le résumé timing final (loggué côté Node après la sync).
|
||||
let aggMonths = 0;
|
||||
let aggBatches = 0;
|
||||
let aggCode15Retries = 0;
|
||||
let aggFallbackCount = 0;
|
||||
|
||||
// Sélectionne Date + Article Code puis pagine le cube. Renvoie le nb de lignes
|
||||
// brutes lues (avant dédoublonnage côté Node).
|
||||
@@ -190,116 +263,155 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
const PAGE = 1400;
|
||||
// Petite latence après chaque mutation de sélection : laisser l'Engine
|
||||
// finaliser l'évaluation de la sélection avant la prochaine requête layout/cube.
|
||||
const SETTLE_MS = 150;
|
||||
const createCubeForBatch = async (): Promise<{ handle: number; id: string }> => {
|
||||
const obj = await rpcWithRetry("createCube", "CreateSessionObject", { qProp: { qInfo: { qType: "cf-net" }, qHyperCubeDef: {
|
||||
const createCubeForBatch = async (timings?: Record<string, number>, onCode15Retry?: () => void): Promise<{ handle: number; id: string }> => {
|
||||
const { value: obj } = await timed("createCube", "CreateSessionObject", () => rpcWithRetry("createCube", "CreateSessionObject", { qProp: { qInfo: { qType: "cf-net" }, qHyperCubeDef: {
|
||||
qDimensions: [{ qLibraryId: dim }],
|
||||
qMeasures: [{ qLibraryId: mca }, { qLibraryId: mqte }, { qLibraryId: mnb }, { qLibraryId: rCamag }, { qLibraryId: rMarge }],
|
||||
qInitialDataFetch: [],
|
||||
} } }, doc);
|
||||
qInitialDataFetch: [{ qTop: 0, qLeft: 0, qWidth: 6, qHeight: PAGE }],
|
||||
} } }, doc, onCode15Retry), timings);
|
||||
const qReturn = obj.qReturn as { qHandle: number; qGenericId?: string; qId?: string };
|
||||
return { handle: qReturn.qHandle, id: String(qReturn.qGenericId ?? qReturn.qId ?? "") };
|
||||
};
|
||||
const destroyCube = async (cube: { handle: number; id: string }) => {
|
||||
const destroyCube = async (cube: { handle: number; id: string }, timings?: Record<string, number>) => {
|
||||
if (!cube.id) return;
|
||||
try {
|
||||
await rpcWithRetry("destroyCube", "DestroySessionObject", { qId: cube.id }, doc);
|
||||
await timed("destroyCube", "DestroySessionObject", () => rpcWithRetry("destroyCube", "DestroySessionObject", { qId: cube.id }, doc), timings);
|
||||
} catch (e) {
|
||||
// Non fatal : l'objet est de toute façon scopé à la session websocket.
|
||||
// Un code 15 ici signifie que l'Engine l'a déjà libéré après un abort.
|
||||
console.log("[qlik-pw] DestroySessionObject ignoré: " + String((e as Error)?.message || e));
|
||||
}
|
||||
};
|
||||
const fetchCubeForSelection = async (sink: Array<Array<string | number>>): Promise<number> => {
|
||||
const cube = await createCubeForBatch();
|
||||
const appendMatrix = (matrix: Array<Array<{ qText?: string; qNum?: number }>>, sink: Array<Array<string | number>>): number => {
|
||||
for (const r of matrix) sink.push([
|
||||
String(r[0]?.qText ?? ""),
|
||||
Number(r[1]?.qNum) || 0,
|
||||
Number(r[2]?.qNum) || 0,
|
||||
Number(r[3]?.qNum) || 0,
|
||||
Number(r[4]?.qNum) || 0,
|
||||
Number(r[5]?.qNum) || 0,
|
||||
]);
|
||||
return matrix.length;
|
||||
};
|
||||
const fetchCubeForSelection = async (sink: Array<Array<string | number>>, timings?: Record<string, number>, onCode15Retry?: () => void): Promise<number> => {
|
||||
const cube = await createCubeForBatch(timings, onCode15Retry);
|
||||
const oh = cube.handle;
|
||||
try {
|
||||
const layout = await rpcWithRetry("getLayout", "GetLayout", {}, oh);
|
||||
const size = ((layout.qLayout as { qHyperCube: { qSize: { qcy: number } } }).qHyperCube).qSize.qcy;
|
||||
const { value: layout } = await timed("layout", "GetLayout", () => rpcWithRetry("layout", "GetLayout", {}, oh, onCode15Retry), timings);
|
||||
const hyperCube = (layout.qLayout as { qHyperCube: { qSize: { qcy: number }; qDataPages?: Array<{ qArea?: { qTop?: number }; qMatrix?: Array<Array<{ qText?: string; qNum?: number }>> }> } }).qHyperCube;
|
||||
const size = hyperCube.qSize.qcy;
|
||||
const firstPage = hyperCube.qDataPages?.find((page) => (page.qArea?.qTop ?? 0) === 0);
|
||||
let got = 0;
|
||||
for (let top = 0; top < size; top += PAGE) {
|
||||
const d = await rpcWithRetry("getHyperCubeData", "GetHyperCubeData", { qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 6, qHeight: PAGE }] }, oh);
|
||||
let top = 0;
|
||||
if (firstPage?.qMatrix?.length) {
|
||||
got += appendMatrix(firstPage.qMatrix, sink);
|
||||
top = PAGE;
|
||||
}
|
||||
for (; top < size; top += PAGE) {
|
||||
const { value: d } = await timed("getCubeData", "GetHyperCubeData", () => rpcWithRetry("getCubeData", "GetHyperCubeData", { qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 6, qHeight: PAGE }] }, oh, onCode15Retry), timings);
|
||||
const matrix = (d.qDataPages as Array<{ qMatrix?: Array<Array<{ qText?: string; qNum?: number }>> }>)?.[0]?.qMatrix ?? [];
|
||||
if (!matrix.length) break;
|
||||
for (const r of matrix) sink.push([
|
||||
String(r[0]?.qText ?? ""),
|
||||
Number(r[1]?.qNum) || 0,
|
||||
Number(r[2]?.qNum) || 0,
|
||||
Number(r[3]?.qNum) || 0,
|
||||
Number(r[4]?.qNum) || 0,
|
||||
Number(r[5]?.qNum) || 0,
|
||||
]);
|
||||
got += matrix.length;
|
||||
got += appendMatrix(matrix, sink);
|
||||
if (matrix.length < PAGE) break;
|
||||
}
|
||||
return got;
|
||||
} finally {
|
||||
await destroyCube(cube);
|
||||
await destroyCube(cube, timings);
|
||||
}
|
||||
};
|
||||
|
||||
if (noDateMode) {
|
||||
// Aucun filtre Date : extraction en un seul appel (tous les codes d'un coup).
|
||||
// On sélectionne uniquement Article Code ; Date reste libre.
|
||||
const timings: Record<string, number> = {};
|
||||
let code15Retries = 0;
|
||||
console.log("mode sans dateFilter — sélection de " + codes.length + " codes en une fois");
|
||||
if (codes.length && fh !== -1) {
|
||||
const sel = await rpc("SelectValues", { qFieldValues: codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh);
|
||||
const { value: sel } = await timed("selectCode", "SelectValues", () => rpc("SelectValues", { qFieldValues: codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh), timings);
|
||||
console.log("SelectValues Article Code → " + JSON.stringify(sel.qReturn));
|
||||
}
|
||||
const got = await fetchCubeForSelection(out);
|
||||
console.log("extraction sans date — " + got + " ligne(s), cumul=" + out.length);
|
||||
const batchRows: Array<Array<string | number>> = [];
|
||||
const { value: got, ms } = await timed("cube", "batch", () => fetchCubeForSelection(batchRows, timings, () => { code15Retries++; }), timings);
|
||||
out.push(...batchRows);
|
||||
aggMonths = 1;
|
||||
aggBatches = 1;
|
||||
aggCode15Retries = code15Retries;
|
||||
console.log("[qlik-pw][timing] noDate rows=" + got + " cumul=" + out.length + " ms=" + ms + " retries15=" + code15Retries + " timings=" + timingSummary(timings));
|
||||
} else {
|
||||
// Itération par mois × lots de codes : pour chaque mois, SelectValues Date
|
||||
// (~30 serials), puis SelectValues Article Code par petits lots. Le cube
|
||||
// reste borné à ~30 jours × 50 codes, sans lire tout le catalogue Qlik.
|
||||
// reste borné à ~30 jours × batch contrôlé, sans lire tout le catalogue Qlik.
|
||||
const total = monthlyPayload.length;
|
||||
const ARTICLE_BATCH = 50;
|
||||
const codeBatches: string[][] = [];
|
||||
if (codes.length) {
|
||||
for (let i = 0; i < codes.length; i += ARTICLE_BATCH) codeBatches.push(codes.slice(i, i + ARTICLE_BATCH));
|
||||
} else {
|
||||
// Comportement historique si aucun code n'est fourni : lire le mois complet.
|
||||
codeBatches.push([]);
|
||||
}
|
||||
console.log("itération par mois — " + total + " mois × " + codeBatches.length + " lot(s) de ≤" + ARTICLE_BATCH + " codes");
|
||||
const fallbackSizes = qlikArticleBatchFallbacks.length ? qlikArticleBatchFallbacks : [150, 75, 50];
|
||||
const fallbackLabel = fallbackSizes.join(">");
|
||||
const estimatedBatches = codes.length ? Math.ceil(codes.length / fallbackSizes[0]) : 1;
|
||||
console.log("itération par mois — " + total + " mois × ~" + estimatedBatches + " lot(s), batchFallbacks=" + fallbackLabel);
|
||||
for (let mi = 0; mi < total; mi++) {
|
||||
const m = monthlyPayload[mi];
|
||||
const prefix = "[qlik-pw] mois " + (mi + 1) + "/" + total + " " + m.label + " (" + m.dateDebut + "→" + m.dateFin + ", " + m.serials.length + "j)";
|
||||
const monthStart = nowMs();
|
||||
const monthTimings: Record<string, number> = {};
|
||||
aggMonths = mi + 1;
|
||||
try {
|
||||
// 1) Sélection Date pour le mois (un seul SelectValues, ~30 valeurs).
|
||||
const dsel = await rpc("SelectValues", {
|
||||
const { value: dsel } = await timed("selectDate", "SelectValues", () => rpc("SelectValues", {
|
||||
qFieldValues: m.serials.map((s) => ({ qNum: s, qText: String(s) })),
|
||||
qToggleMode: false,
|
||||
qSoftLock: true,
|
||||
}, dfh);
|
||||
}, dfh), monthTimings);
|
||||
console.log(prefix + " — SelectValues Date " + m.serials.length + " serials → " + JSON.stringify(dsel.qReturn));
|
||||
// Laisser l'Engine finaliser l'évaluation de la sélection Date
|
||||
// avant la première SelectValues Article Code du 1er lot.
|
||||
await sleep(SETTLE_MS);
|
||||
await sleep(qlikSettleMs);
|
||||
|
||||
// 2) Sélection Article Code par lots contrôlés.
|
||||
for (let bi = 0; bi < codeBatches.length; bi++) {
|
||||
const batch = codeBatches[bi];
|
||||
const batchPrefix = prefix + " — lot " + (bi + 1) + "/" + codeBatches.length + " (" + batch.length + " codes)";
|
||||
let codeIndex = 0;
|
||||
let batchNumber = 0;
|
||||
let fallbackIndex = 0;
|
||||
const totalCodes = codes.length || 0;
|
||||
while (codes.length ? codeIndex < totalCodes : batchNumber === 0) {
|
||||
const batchSize = codes.length ? fallbackSizes[fallbackIndex] : 0;
|
||||
const batch = codes.length ? codes.slice(codeIndex, Math.min(codeIndex + batchSize, totalCodes)) : [];
|
||||
const batchStart = codeIndex;
|
||||
const batchEnd = codeIndex + batch.length;
|
||||
const timings: Record<string, number> = {};
|
||||
let code15Retries = 0;
|
||||
batchNumber++;
|
||||
const batchPrefix = prefix + " — lot " + batchNumber + " [" + batchStart + "," + batchEnd + ") size=" + batch.length;
|
||||
try {
|
||||
if (fh !== -1) {
|
||||
await rpc("Clear", {}, fh);
|
||||
await timed("clearCode", "Clear", () => rpc("Clear", {}, fh), timings);
|
||||
// Laisser l'Engine appliquer le Clear avant la nouvelle sélection.
|
||||
await sleep(SETTLE_MS);
|
||||
const csel = await rpc("SelectValues", {
|
||||
await sleep(qlikSettleMs);
|
||||
const { value: csel } = await timed("selectCode", "SelectValues", () => rpc("SelectValues", {
|
||||
qFieldValues: batch.map((c) => ({ qText: c })),
|
||||
qToggleMode: false,
|
||||
qSoftLock: true,
|
||||
}, fh);
|
||||
}, fh), timings);
|
||||
console.log(batchPrefix + " — SelectValues Article Code → " + JSON.stringify(csel.qReturn));
|
||||
// Laisser l'Engine finaliser l'évaluation de la sélection
|
||||
// avant de créer / interroger le cube (évite code 15 sur le 1er lot).
|
||||
await sleep(SETTLE_MS);
|
||||
await sleep(qlikSettleMs);
|
||||
}
|
||||
const got = await fetchCubeForSelection(out);
|
||||
console.log(batchPrefix + " — " + got + " ligne(s), cumul=" + out.length);
|
||||
const batchRows: Array<Array<string | number>> = [];
|
||||
const { value: got, ms } = await timed("cube", "batch", () => fetchCubeForSelection(batchRows, timings, () => { code15Retries++; }), timings);
|
||||
out.push(...batchRows);
|
||||
aggBatches++;
|
||||
aggCode15Retries += code15Retries;
|
||||
console.log("[qlik-pw][timing] " + batchPrefix + " — rows=" + got + " cumul=" + out.length + " ms=" + ms + " retries15=" + code15Retries + " timings=" + timingSummary(timings));
|
||||
if (!codes.length) break;
|
||||
codeIndex = batchEnd;
|
||||
fallbackIndex = 0;
|
||||
} catch (batchErr) {
|
||||
const msg = String((batchErr as Error)?.message || batchErr);
|
||||
const nextFallbackIndex = fallbackIndex + 1;
|
||||
const nextSize = fallbackSizes[nextFallbackIndex];
|
||||
if (codes.length && isEngineCode15(batchErr) && nextSize && batch.length > nextSize) {
|
||||
console.log("[qlik-pw][timing] " + batchPrefix + " — fallback code15 " + batch.length + "→" + nextSize + " retries15=" + code15Retries + " timings=" + timingSummary(timings));
|
||||
fallbackIndex = nextFallbackIndex;
|
||||
aggFallbackCount++;
|
||||
continue;
|
||||
}
|
||||
console.error(batchPrefix + " — ABORT — " + msg);
|
||||
throw new Error(batchPrefix + " — " + msg);
|
||||
}
|
||||
@@ -312,17 +424,18 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
// 3) Clear Date + Article Code avant le mois suivant (soulage l'Engine).
|
||||
if (mi < total - 1) {
|
||||
try {
|
||||
await rpc("Clear", {}, dfh);
|
||||
if (fh !== -1) await rpc("Clear", {}, fh);
|
||||
await timed("clearDate", "Clear", () => rpc("Clear", {}, dfh), monthTimings);
|
||||
if (fh !== -1) await timed("clearCodeMonth", "Clear", () => rpc("Clear", {}, fh), monthTimings);
|
||||
} catch (clearErr) {
|
||||
console.log(prefix + " — Clear ignoré: " + String((clearErr as Error)?.message || clearErr));
|
||||
}
|
||||
}
|
||||
console.log("[qlik-pw][timing] " + prefix + " — done ms=" + Math.round(nowMs() - monthStart) + " cumul=" + out.length + " timings=" + timingSummary(monthTimings));
|
||||
}
|
||||
}
|
||||
|
||||
ws.close();
|
||||
resolve({ ok: true, rows: out, size: out.length });
|
||||
resolve({ ok: true, rows: out, size: out.length, months: aggMonths, batches: aggBatches, code15Retries: aggCode15Retries, fallbackCount: aggFallbackCount });
|
||||
} catch (e) { try { ws.close(); } catch { /* noop */ } fail(String((e as Error)?.message || e)); }
|
||||
})();
|
||||
}),
|
||||
@@ -338,11 +451,22 @@ export async function fetchNetworkMetricsPlaywright(
|
||||
token: csrfToken,
|
||||
monthlyPayload,
|
||||
noDateMode: !dateFilter,
|
||||
qlikSettleMs,
|
||||
qlikCode15MaxAttempts,
|
||||
qlikArticleBatchFallbacks,
|
||||
},
|
||||
)) as InPageResult;
|
||||
|
||||
if (!result.ok) throw new Error(`[qlik-pw] ${result.error}`);
|
||||
|
||||
// Résumé timing final : synthèse compacte de la sync pour audit/debug.
|
||||
// Les logs détaillés par étape restent en place (timings batch/mois) ;
|
||||
// ici on consolide les compteurs agrégés et la durée totale.
|
||||
const totalMs = Date.now() - totalStart;
|
||||
console.log(
|
||||
`[qlik-pw][summary] rows=${result.size ?? 0} months=${result.months ?? 0} batches=${result.batches ?? 0} code15Retries=${result.code15Retries ?? 0} fallbackCount=${result.fallbackCount ?? 0} totalMs=${totalMs}`,
|
||||
);
|
||||
|
||||
const wanted = codeCentraux ? new Set(codeCentraux) : null;
|
||||
const out = new Map<string, NetworkMetric>();
|
||||
const margeWeight = new Map<string, number>();
|
||||
|
||||
Reference in new issue
Block a user