From 6cdb845a1f38d538bc6dd43c5bbf4452613db4c9 Mon Sep 17 00:00:00 2001 From: Michael Date: Sun, 21 Jun 2026 00:37:04 +0200 Subject: [PATCH] =?UTF-8?q?fix(qlik):=20use=20qlik-csrf-token=20+=20real?= =?UTF-8?q?=20engine=20qIds=20=E2=80=94=20extraction=20works?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Validated end-to-end. The recent Qlik client authorizes the engine websocket via ?reloadUri=...&qlik-csrf-token= (not Xrfkey). Capture that token from the client's own ws, then open ours in-page with it. Use the real engine master-item qIds (dim Article Code=yesCP, CA N=JhqJ, Quantité N=41516861..., Magasin Ventes Nb N=yNBLjc) — the QRS object IDs returned error 7001. Select the supplier's codes on field "Article Code" (the app holds ~1.2M codes). Co-Authored-By: Claude Opus 4.8 --- src/lib/qlik-client.ts | 9 +++-- src/lib/qlik-playwright.ts | 78 +++++++++++++++++++------------------- 2 files changed, 43 insertions(+), 44 deletions(-) diff --git a/src/lib/qlik-client.ts b/src/lib/qlik-client.ts index 6181bb0..d3628e2 100644 --- a/src/lib/qlik-client.ts +++ b/src/lib/qlik-client.ts @@ -62,10 +62,11 @@ export function getQlikConfig(): QlikConfig { domain: process.env.QLIK_DOMAIN ?? "", workstation: process.env.QLIK_WORKSTATION ?? "", appNetwork: process.env.QLIK_APP_NETWORK ?? "9872ee6e-d64a-4b43-984a-076bf1f7f647", - dimCodeArticleId: process.env.QLIK_DIM_CODE_ARTICLE_ID ?? "fcd239e5-288b-4830-a047-0e3d7665d971", - measCaId: process.env.QLIK_MEAS_CA_ID ?? "43a76088-86fa-402e-a80e-0efd7701b3e1", - measQteId: process.env.QLIK_MEAS_QTE_ID ?? "7b40caf1-be4b-4811-8d45-50acde33e715", - measNbMagId: process.env.QLIK_MEAS_NBMAG_ID ?? "8b63fae5-db2f-4e4c-8618-f3e9d60b6b3b", + // qIds engine (master items de l'app "Magasins Vision Consolidée") + dimCodeArticleId: process.env.QLIK_DIM_CODE_ARTICLE_ID ?? "yesCP", + measCaId: process.env.QLIK_MEAS_CA_ID ?? "JhqJ", + measQteId: process.env.QLIK_MEAS_QTE_ID ?? "41516861-5997-4635-8187-3643a2bde422", + measNbMagId: process.env.QLIK_MEAS_NBMAG_ID ?? "yNBLjc", tlsInsecure: (process.env.QLIK_TLS_INSECURE ?? "true") === "true", timeoutMs: Number(process.env.QLIK_TIMEOUT_MS ?? "60000"), }; diff --git a/src/lib/qlik-playwright.ts b/src/lib/qlik-playwright.ts index 59e30f2..823c19e 100644 --- a/src/lib/qlik-playwright.ts +++ b/src/lib/qlik-playwright.ts @@ -2,15 +2,13 @@ * CollectFlow — Extraction Qlik via Playwright (Chromium headless). * * Le proxy Qlik refuse un websocket "raw" côté serveur (403). On ouvre donc - * l'app dans Chromium avec une **session authentifiée** (cookie X-Qlik-Session - * injecté, obtenu via le flux ticket NTLM côté Node) puis on ouvre le websocket - * Engine **in-page** (même origine + cookie navigateur = accepté par le proxy). - * - * L'API Engine (wss://host/app/) est stable quelle que soit la version du - * client Qlik (ici qmfe/single-spa, sans l'ancienne Capability API js/qlik). + * l'app dans Chromium avec une session authentifiée (cookie X-Qlik-Session + * injecté via le flux ticket NTLM), on récupère le **qlik-csrf-token** que le + * client Qlik utilise pour SON websocket, puis on ouvre notre propre websocket + * Engine **in-page** avec ce token (le proxy l'accepte). * * Efficacité : sélection des codes du fournisseur sur le champ "Article Code" - * → l'hypercube ne renvoie que ces lignes. + * (l'app contient ~1,2M codes) → l'hypercube ne renvoie que ces lignes. */ import "server-only"; @@ -32,13 +30,7 @@ async function getBrowser(): Promise { return browserPromise; } -interface InPageResult { - ok: boolean; - rows?: Array>; - size?: number; - error?: string; - diag?: Record; -} +interface InPageResult { ok: boolean; rows?: Array>; size?: number; error?: string; diag?: Record; } export async function fetchNetworkMetricsPlaywright( codeCentraux?: string[], @@ -47,7 +39,6 @@ export async function fetchNetworkMetricsPlaywright( if (!cfg.appNetwork) throw new Error("[qlik-pw] QLIK_APP_NETWORK manquant"); if (!cfg.user || !cfg.password) throw new Error("[qlik-pw] identifiants Qlik manquants"); - // 1. Session Qlik fiable (flux ticket NTLM côté Node) → cookie const sess = await qlikNtlmSession(cfg); const [cookieName, ...rest] = sess.cookie.split("="); const cookieValue = rest.join("="); @@ -61,31 +52,41 @@ export async function fetchNetworkMetricsPlaywright( try { const page = await ctx.newPage(); - await page.goto(`https://${cfg.host}/sense/app/${cfg.appNetwork}`, { waitUntil: "domcontentloaded", timeout: cfg.timeoutMs }) - .catch(() => { /* le ws in-page suffit */ }); - const xrf = sess.xrfkey; + // Capture le qlik-csrf-token depuis le websocket que le client Qlik ouvre + let csrfToken: string | null = null; + let resolveToken: (() => void) | null = null; + const tokenReady = new Promise((r) => { resolveToken = r; }); + page.on("websocket", (ws) => { + const m = ws.url().match(/qlik-csrf-token=([^&]+)/); + if (m && !csrfToken) { csrfToken = decodeURIComponent(m[1]); resolveToken?.(); } + }); + + await page.goto(`https://${cfg.host}/sense/app/${cfg.appNetwork}`, { waitUntil: "domcontentloaded", timeout: cfg.timeoutMs }) + .catch(() => { /* le client ouvre son ws ensuite */ }); + await Promise.race([tokenReady, page.waitForTimeout(15000)]); + if (!csrfToken) throw new Error("[qlik-pw] qlik-csrf-token introuvable (client Qlik non chargé ?)"); + const result = (await page.evaluate( - ({ app, dim, mca, mqte, mnb, codes, xrf }: { app: string; dim: string; mca: string; mqte: string; mnb: string; codes: string[]; xrf: string }) => + ({ app, dim, mca, mqte, mnb, codes, token }: { app: string; dim: string; mca: string; mqte: string; mnb: string; codes: string[]; token: string }) => new Promise((resolve) => { const loc = (window as unknown as { location: Location }).location; - const diag: Record = { href: loc.href }; - const ws = new WebSocket(`wss://${loc.host}/app/${app}?Xrfkey=${xrf}`); + const url = `wss://${loc.host}/app/${app}?reloadUri=${encodeURIComponent(loc.href)}&qlik-csrf-token=${token}`; + const ws = new WebSocket(url); let id = 0; - const pend = new Map void; rej: (e: unknown) => void }>(); + const pend = new Map void; rej: (e: unknown) => void }>(); const rpc = (method: string, params: unknown, handle = -1) => new Promise<{ [k: string]: unknown }>((res, rej) => { - const i = ++id; pend.set(i, { res: res as (v: unknown) => void, rej }); + const i = ++id; pend.set(i, { res, rej }); ws.send(JSON.stringify({ jsonrpc: "2.0", id: i, handle, method, params })); }); - const fail = (error: string) => resolve({ ok: false, error, diag }); + const fail = (error: string) => resolve({ ok: false, error }); const ready = new Promise((res, rej) => { ws.onmessage = (ev: MessageEvent) => { const m = JSON.parse(ev.data as string); if (m.method === "OnConnected") return res(); if (m.id && pend.has(m.id)) { const x = pend.get(m.id)!; pend.delete(m.id); m.error ? x.rej(new Error(JSON.stringify(m.error))) : x.res(m.result); } }; - ws.onclose = (ev: CloseEvent) => { diag.wsClose = { code: ev.code, reason: ev.reason }; }; - ws.onerror = () => rej(new Error("ws error (403 proxy ?)")); + ws.onerror = () => rej(new Error("ws error (403 ?)")); setTimeout(() => rej(new Error("ws timeout")), 30000); }); (async () => { @@ -93,18 +94,16 @@ export async function fetchNetworkMetricsPlaywright( await ready; const open = await rpc("OpenDoc", { qDocName: app }); const doc = (open.qReturn as { qHandle: number }).qHandle; - // Sélection des codes du fournisseur (efficacité) — best-effort + // Sélection des codes du fournisseur (sinon ~1,2M lignes) if (codes.length) { - try { - const gd = await rpc("GetField", { qFieldName: "Article Code" }, doc); - const fh = (gd.qReturn as { qHandle: number }).qHandle; - await rpc("SelectValues", { qFieldValues: codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh); - } catch { /* noop */ } + const gf = await rpc("GetField", { qFieldName: "Article Code" }, doc); + const fh = (gf.qReturn as { qHandle: number }).qHandle; + await rpc("SelectValues", { qFieldValues: codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh); } const obj = await rpc("CreateSessionObject", { qProp: { qInfo: { qType: "cf-net" }, qHyperCubeDef: { - qDimensions: [{ qLibraryId: dim, qNullSuppression: true }], + qDimensions: [{ qLibraryId: dim }], qMeasures: [{ qLibraryId: mca }, { qLibraryId: mqte }, { qLibraryId: mnb }], - qInitialDataFetch: [], qSuppressMissing: true, + qInitialDataFetch: [], } } }, doc); const oh = (obj.qReturn as { qHandle: number }).qHandle; const layout = await rpc("GetLayout", {}, oh); @@ -119,25 +118,24 @@ export async function fetchNetworkMetricsPlaywright( if (matrix.length < PAGE) break; } ws.close(); - resolve({ ok: true, rows: out, size, diag }); + resolve({ ok: true, rows: out, size }); } catch (e) { try { ws.close(); } catch { /* noop */ } fail(String((e as Error)?.message || e)); } })(); }), - { app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId, codes: codeCentraux ?? [], xrf }, + { app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId, codes: codeCentraux ?? [], token: csrfToken }, )) as InPageResult; - console.log(`[qlik-pw] diag: ${JSON.stringify(result.diag)}`); - if (!result.ok) throw new Error(`[qlik-pw] ${result.error} | diag=${JSON.stringify(result.diag)}`); + if (!result.ok) throw new Error(`[qlik-pw] ${result.error}`); const wanted = codeCentraux ? new Set(codeCentraux) : null; const out = new Map(); for (const r of result.rows ?? []) { const code = String(r[0]).trim(); - if (!code) continue; + if (!code || code === "-") continue; if (wanted && !wanted.has(code)) continue; out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0 }); } - console.log(`[qlik-pw] ${out.size} produits réseau (cube size ${result.size})`); + console.log(`[qlik-pw] ${out.size} produits réseau (cube ${result.size})`); return out; } finally { await ctx.close();