From 694b7801604b1eaad87ddacd753fb98091871a77 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 20 Jun 2026 23:54:03 +0200 Subject: [PATCH] feat(qlik): extract network data via Playwright (in-page ws) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Qlik proxy refuses a raw server-side websocket (403, even internally), so extraction now runs through headless Chromium (NTLM via httpCredentials, like the hermes agent) and opens the Engine websocket in-page — same origin, which the proxy accepts. Efficient: selects the supplier's article codes on the "Article Code" field so the hypercube returns only those rows. - qlik-playwright.ts: browser singleton, in-page hypercube extraction (paginated) - /api/qlik/sync: use the Playwright extractor - Dockerfile: install chromium + headless deps, PLAYWRIGHT_CHROMIUM_PATH, copy playwright-core - deps: playwright-core (lockfiles synced) Co-Authored-By: Claude Opus 4.8 --- Dockerfile | 7 ++ next.config.ts | 2 +- package-lock.json | 13 +++ package.json | 1 + pnpm-lock.yaml | 10 +++ src/app/api/qlik/sync/route.ts | 4 +- src/lib/qlik-playwright.ts | 143 +++++++++++++++++++++++++++++++++ 7 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 src/lib/qlik-playwright.ts diff --git a/Dockerfile b/Dockerfile index 9859f90..c75a901 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,6 +32,10 @@ ENV NODE_ENV=production # Next.js telemetry is disabled ENV NEXT_TELEMETRY_DISABLED=1 +# Chromium pour l'extraction Qlik via Playwright (ws in-page) + dépendances headless +RUN apk add --no-cache chromium nss freetype harfbuzz ca-certificates ttf-freefont +ENV PLAYWRIGHT_CHROMIUM_PATH=/usr/bin/chromium-browser + RUN addgroup --system --gid 1001 nodejs RUN adduser --system --uid 1001 nextjs @@ -46,6 +50,9 @@ RUN chown nextjs:nodejs .next data COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./ COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static +# playwright-core n'est pas toujours tracé par le standalone — on le copie explicitement +COPY --from=deps --chown=nextjs:nodejs /app/node_modules/playwright-core ./node_modules/playwright-core + # Grant write access to /app for settings persistence RUN chown -R nextjs:nodejs /app diff --git a/next.config.ts b/next.config.ts index 63ddf4e..c05866c 100644 --- a/next.config.ts +++ b/next.config.ts @@ -1,7 +1,7 @@ /** @type {import('next').NextConfig} */ const nextConfig = { output: "standalone", - serverExternalPackages: ["pg", "ws", "httpntlm"], + serverExternalPackages: ["pg", "ws", "httpntlm", "playwright-core"], experimental: { serverActions: { bodySizeLimit: "50mb", diff --git a/package-lock.json b/package-lock.json index 7bdeec4..055ac8c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -26,6 +26,7 @@ "next-auth": "^5.0.0-beta.30", "next-themes": "^0.4.6", "pg": "^8.18.0", + "playwright-core": "^1.61.0", "radix-ui": "^1.4.3", "react": "19.2.3", "react-dom": "19.2.3", @@ -13285,6 +13286,18 @@ "node": ">=16.20.0" } }, + "node_modules/playwright-core": { + "version": "1.61.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz", + "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/possible-typed-array-names": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz", diff --git a/package.json b/package.json index 7698253..e92328e 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "next-auth": "^5.0.0-beta.30", "next-themes": "^0.4.6", "pg": "^8.18.0", + "playwright-core": "^1.61.0", "radix-ui": "^1.4.3", "react": "19.2.3", "react-dom": "19.2.3", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index dbcfc38..833c28d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: pg: specifier: ^8.18.0 version: 8.20.0 + playwright-core: + specifier: ^1.61.0 + version: 1.61.0 radix-ui: specifier: ^1.4.3 version: 1.4.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3) @@ -4503,6 +4506,11 @@ packages: resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==} engines: {node: '>=16.20.0'} + playwright-core@1.61.0: + resolution: {integrity: sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==} + engines: {node: '>=18'} + hasBin: true + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -9931,6 +9939,8 @@ snapshots: pkce-challenge@5.0.1: {} + playwright-core@1.61.0: {} + possible-typed-array-names@1.1.0: {} postcss-selector-parser@7.1.1: diff --git a/src/app/api/qlik/sync/route.ts b/src/app/api/qlik/sync/route.ts index a8f131f..bf67e8f 100644 --- a/src/app/api/qlik/sync/route.ts +++ b/src/app/api/qlik/sync/route.ts @@ -1,6 +1,6 @@ import { NextRequest, NextResponse } from "next/server"; import { auth } from "@/lib/auth"; -import { fetchNetworkMetrics } from "@/lib/qlik-client"; +import { fetchNetworkMetricsPlaywright } from "@/lib/qlik-playwright"; import { upsertNetworkMetrics } from "@/lib/qlik-network-cache"; import { pgGetArticlesByFournisseur } from "@/lib/pg-ff-client"; @@ -44,7 +44,7 @@ export async function POST(req: NextRequest) { }); } - const metrics = await fetchNetworkMetrics(codes); + const metrics = await fetchNetworkMetricsPlaywright(codes); const count = await upsertNetworkMetrics([...metrics.values()]); return NextResponse.json({ success: true, diff --git a/src/lib/qlik-playwright.ts b/src/lib/qlik-playwright.ts new file mode 100644 index 0000000..9613a3b --- /dev/null +++ b/src/lib/qlik-playwright.ts @@ -0,0 +1,143 @@ +/** + * CollectFlow — Extraction Qlik via Playwright (Chromium headless). + * + * Le proxy Qlik refuse un websocket "raw" côté serveur (403), mais l'accepte + * depuis une vraie session navigateur. On ouvre donc l'app dans Chromium + * (auth NTLM via httpCredentials, comme l'agent hermes), puis on extrait + * l'hypercube via un websocket Engine **in-page** (même origine = accepté). + * + * Efficacité : on applique une sélection sur le champ "Article Code" = codes + * du fournisseur → l'hypercube ne renvoie que ces lignes (pull minimal). + */ + +import "server-only"; +import { chromium, type Browser } from "playwright-core"; +import { getQlikConfig, type QlikConfig, type NetworkMetric } from "@/lib/qlik-client"; + +let browserPromise: Promise | null = null; + +async function getBrowser(): Promise { + if (!browserPromise) { + const execPath = process.env.PLAYWRIGHT_CHROMIUM_PATH; + browserPromise = chromium.launch({ + headless: true, + executablePath: execPath || undefined, + channel: execPath ? undefined : "chrome", + args: ["--no-sandbox", "--disable-dev-shm-usage", "--ignore-certificate-errors"], + }).catch((e) => { browserPromise = null; throw e; }); + } + return browserPromise; +} + +interface InPageParams { + app: string; dim: string; mca: string; mqte: string; mnb: string; + codes: string[]; xrf: string; +} +interface InPageResult { ok: boolean; rows?: Array>; size?: number; error?: string; } + +/** Code exécuté DANS la page (contexte navigateur) — ws Engine in-page. */ +function inPageExtract(p: InPageParams): Promise { + // @ts-expect-error exécuté dans le navigateur + return (async () => { + const ws = new WebSocket(`wss://${location.host}/app/${p.app}?Xrfkey=${p.xrf}`); + let id = 0; + const pend = new Map void; rej: (e: any) => void }>(); + const rpc = (method: string, params: any, handle = -1) => new Promise((res, rej) => { + const i = ++id; pend.set(i, { res, rej }); + ws.send(JSON.stringify({ jsonrpc: "2.0", id: i, handle, method, params })); + }); + const ready = new Promise((res, rej) => { + ws.onmessage = (ev: MessageEvent) => { + const m = JSON.parse(ev.data as string); + if (m.method === "OnConnected") return res(); + if (m.id && pend.has(m.id)) { const x = pend.get(m.id)!; pend.delete(m.id); m.error ? x.rej(new Error(JSON.stringify(m.error))) : x.res(m.result); } + }; + ws.onerror = () => rej(new Error("ws error (403 proxy ?)")); + setTimeout(() => rej(new Error("ws timeout")), 30000); + }); + try { + await ready; + const open = await rpc("OpenDoc", { qDocName: p.app }); + const doc = open.qReturn.qHandle; + + // Sélection des codes du fournisseur sur le champ derrière "Article Code" (efficacité) + if (p.codes.length) { + try { + const gd = await rpc("GetDimension", { qId: p.dim }, doc); + const dh = gd.qReturn.qHandle; + const dl = await rpc("GetLayout", {}, dh); + const field = dl.qLayout?.qDim?.qFieldDefs?.[0]; + if (field) { + const gf = await rpc("GetField", { qFieldName: field }, doc); + const fh = gf.qReturn.qHandle; + await rpc("SelectValues", { qFieldValues: p.codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh); + } + } catch { /* sélection best-effort */ } + } + + const obj = await rpc("CreateSessionObject", { qProp: { qInfo: { qType: "cf-network" }, qHyperCubeDef: { + qDimensions: [{ qLibraryId: p.dim, qNullSuppression: true }], + qMeasures: [{ qLibraryId: p.mca }, { qLibraryId: p.mqte }, { qLibraryId: p.mnb }], + qInitialDataFetch: [], qSuppressMissing: true, + } } }, doc); + const oh = obj.qReturn.qHandle; + const layout = await rpc("GetLayout", {}, oh); + const size = layout.qLayout.qHyperCube.qSize.qcy as number; + + const out: Array> = []; + const PAGE = 2500; + for (let top = 0; top < size; top += PAGE) { + const d = await rpc("GetHyperCubeData", { qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 4, qHeight: PAGE }] }, oh); + const matrix = d.qDataPages?.[0]?.qMatrix ?? []; + if (!matrix.length) break; + for (const r of matrix) out.push([String(r[0].qText ?? ""), Number(r[1].qNum) || 0, Number(r[2].qNum) || 0, Number(r[3].qNum) || 0]); + if (matrix.length < PAGE) break; + } + ws.close(); + return { ok: true, rows: out, size }; + } catch (e: any) { + try { ws.close(); } catch {} + return { ok: false, error: String(e?.message || e) }; + } + })(); +} + +export async function fetchNetworkMetricsPlaywright( + codeCentraux?: string[], + cfg: QlikConfig = getQlikConfig(), +): Promise> { + if (!cfg.appNetwork) throw new Error("[qlik-pw] QLIK_APP_NETWORK manquant"); + if (!cfg.user || !cfg.password) throw new Error("[qlik-pw] identifiants Qlik manquants"); + + const xrf = Array.from({ length: 16 }, () => "abcdefghijklmnopqrstuvwxyz0123456789"[Math.floor(Math.random() * 36)]).join(""); + const browser = await getBrowser(); + const ctx = await browser.newContext({ + httpCredentials: { username: cfg.user, password: cfg.password, origin: `https://${cfg.host}` }, + ignoreHTTPSErrors: true, + }); + try { + const page = await ctx.newPage(); + await page.goto(`https://${cfg.host}/sense/app/${cfg.appNetwork}`, { waitUntil: "domcontentloaded", timeout: cfg.timeoutMs }) + .catch(() => { /* la page peut ne pas finir de charger, le ws in-page suffit */ }); + + const result = (await page.evaluate(inPageExtract as never, { + app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId, + codes: codeCentraux ?? [], xrf, + })) as InPageResult; + + if (!result.ok) throw new Error(`[qlik-pw] extraction: ${result.error}`); + + const wanted = codeCentraux ? new Set(codeCentraux) : null; + const out = new Map(); + for (const r of result.rows ?? []) { + const code = String(r[0]).trim(); + if (!code) continue; + if (wanted && !wanted.has(code)) continue; + out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0 }); + } + console.log(`[qlik-pw] ${out.size} produits réseau (cube size ${result.size})`); + return out; + } finally { + await ctx.close(); + } +}