feat(qlik): extract network data via Playwright (in-page ws)

The Qlik proxy refuses a raw server-side websocket (403, even internally), so
extraction now runs through headless Chromium (NTLM via httpCredentials, like
the hermes agent) and opens the Engine websocket in-page — same origin, which
the proxy accepts. Efficient: selects the supplier's article codes on the
"Article Code" field so the hypercube returns only those rows.

- qlik-playwright.ts: browser singleton, in-page hypercube extraction (paginated)
- /api/qlik/sync: use the Playwright extractor
- Dockerfile: install chromium + headless deps, PLAYWRIGHT_CHROMIUM_PATH, copy playwright-core
- deps: playwright-core (lockfiles synced)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Dockerfile

@@ -32,6 +32,10 @@ ENV NODE_ENV=production
 # Next.js telemetry is disabled
 ENV NEXT_TELEMETRY_DISABLED=1
 
+# Chromium pour l'extraction Qlik via Playwright (ws in-page) + dépendances headless
+RUN apk add --no-cache chromium nss freetype harfbuzz ca-certificates ttf-freefont
+ENV PLAYWRIGHT_CHROMIUM_PATH=/usr/bin/chromium-browser
+
 RUN addgroup --system --gid 1001 nodejs
 RUN adduser --system --uid 1001 nextjs
 
@@ -46,6 +50,9 @@ RUN chown nextjs:nodejs .next data
 COPY --from=builder --chown=nextjs:nodejs /app/.next/standalone ./
 COPY --from=builder --chown=nextjs:nodejs /app/.next/static ./.next/static
 
+# playwright-core n'est pas toujours tracé par le standalone — on le copie explicitement
+COPY --from=deps --chown=nextjs:nodejs /app/node_modules/playwright-core ./node_modules/playwright-core
+
 # Grant write access to /app for settings persistence
 RUN chown -R nextjs:nodejs /app
 

next.config.ts

@@ -1,7 +1,7 @@
 /** @type {import('next').NextConfig} */
 const nextConfig = {
   output: "standalone",
-  serverExternalPackages: ["pg", "ws", "httpntlm"],
+  serverExternalPackages: ["pg", "ws", "httpntlm", "playwright-core"],
   experimental: {
     serverActions: {
       bodySizeLimit: "50mb",

package-lock.json

@@ -26,6 +26,7 @@
         "next-auth": "^5.0.0-beta.30",
         "next-themes": "^0.4.6",
         "pg": "^8.18.0",
+        "playwright-core": "^1.61.0",
         "radix-ui": "^1.4.3",
         "react": "19.2.3",
         "react-dom": "19.2.3",
@@ -13285,6 +13286,18 @@
         "node": ">=16.20.0"
       }
     },
+    "node_modules/playwright-core": {
+      "version": "1.61.0",
+      "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.0.tgz",
+      "integrity": "sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==",
+      "license": "Apache-2.0",
+      "bin": {
+        "playwright-core": "cli.js"
+      },
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/possible-typed-array-names": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",

package.json

@@ -30,6 +30,7 @@
     "next-auth": "^5.0.0-beta.30",
     "next-themes": "^0.4.6",
     "pg": "^8.18.0",
+    "playwright-core": "^1.61.0",
     "radix-ui": "^1.4.3",
     "react": "19.2.3",
     "react-dom": "19.2.3",

pnpm-lock.yaml

@@ -62,6 +62,9 @@ importers:
       pg:
         specifier: ^8.18.0
         version: 8.20.0
+      playwright-core:
+        specifier: ^1.61.0
+        version: 1.61.0
       radix-ui:
         specifier: ^1.4.3
         version: 1.4.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.3(react@19.2.3))(react@19.2.3)
@@ -4503,6 +4506,11 @@ packages:
     resolution: {integrity: sha512-wQ0b/W4Fr01qtpHlqSqspcj3EhBvimsdh0KlHhH8HRZnMsEa0ea2fTULOXOS9ccQr3om+GcGRk4e+isrZWV8qQ==}
     engines: {node: '>=16.20.0'}
 
+  playwright-core@1.61.0:
+    resolution: {integrity: sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==}
+    engines: {node: '>=18'}
+    hasBin: true
+
   possible-typed-array-names@1.1.0:
     resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==}
     engines: {node: '>= 0.4'}
@@ -9931,6 +9939,8 @@ snapshots:
 
   pkce-challenge@5.0.1: {}
 
+  playwright-core@1.61.0: {}
+
   possible-typed-array-names@1.1.0: {}
 
   postcss-selector-parser@7.1.1:

src/app/api/qlik/sync/route.ts

@@ -1,6 +1,6 @@
 import { NextRequest, NextResponse } from "next/server";
 import { auth } from "@/lib/auth";
-import { fetchNetworkMetrics } from "@/lib/qlik-client";
+import { fetchNetworkMetricsPlaywright } from "@/lib/qlik-playwright";
 import { upsertNetworkMetrics } from "@/lib/qlik-network-cache";
 import { pgGetArticlesByFournisseur } from "@/lib/pg-ff-client";
 
@@ -44,7 +44,7 @@ export async function POST(req: NextRequest) {
             });
         }
 
-        const metrics = await fetchNetworkMetrics(codes);
+        const metrics = await fetchNetworkMetricsPlaywright(codes);
         const count = await upsertNetworkMetrics([...metrics.values()]);
         return NextResponse.json({
             success: true,

src/lib/qlik-playwright.ts

@@ -0,0 +1,143 @@
+/**
+ * CollectFlow — Extraction Qlik via Playwright (Chromium headless).
+ *
+ * Le proxy Qlik refuse un websocket "raw" côté serveur (403), mais l'accepte
+ * depuis une vraie session navigateur. On ouvre donc l'app dans Chromium
+ * (auth NTLM via httpCredentials, comme l'agent hermes), puis on extrait
+ * l'hypercube via un websocket Engine **in-page** (même origine = accepté).
+ *
+ * Efficacité : on applique une sélection sur le champ "Article Code" = codes
+ * du fournisseur → l'hypercube ne renvoie que ces lignes (pull minimal).
+ */
+
+import "server-only";
+import { chromium, type Browser } from "playwright-core";
+import { getQlikConfig, type QlikConfig, type NetworkMetric } from "@/lib/qlik-client";
+
+let browserPromise: Promise<Browser> | null = null;
+
+async function getBrowser(): Promise<Browser> {
+    if (!browserPromise) {
+        const execPath = process.env.PLAYWRIGHT_CHROMIUM_PATH;
+        browserPromise = chromium.launch({
+            headless: true,
+            executablePath: execPath || undefined,
+            channel: execPath ? undefined : "chrome",
+            args: ["--no-sandbox", "--disable-dev-shm-usage", "--ignore-certificate-errors"],
+        }).catch((e) => { browserPromise = null; throw e; });
+    }
+    return browserPromise;
+}
+
+interface InPageParams {
+    app: string; dim: string; mca: string; mqte: string; mnb: string;
+    codes: string[]; xrf: string;
+}
+interface InPageResult { ok: boolean; rows?: Array<Array<string | number>>; size?: number; error?: string; }
+
+/** Code exécuté DANS la page (contexte navigateur) — ws Engine in-page. */
+function inPageExtract(p: InPageParams): Promise<InPageResult> {
+    // @ts-expect-error exécuté dans le navigateur
+    return (async () => {
+        const ws = new WebSocket(`wss://${location.host}/app/${p.app}?Xrfkey=${p.xrf}`);
+        let id = 0;
+        const pend = new Map<number, { res: (v: any) => void; rej: (e: any) => void }>();
+        const rpc = (method: string, params: any, handle = -1) => new Promise<any>((res, rej) => {
+            const i = ++id; pend.set(i, { res, rej });
+            ws.send(JSON.stringify({ jsonrpc: "2.0", id: i, handle, method, params }));
+        });
+        const ready = new Promise<void>((res, rej) => {
+            ws.onmessage = (ev: MessageEvent) => {
+                const m = JSON.parse(ev.data as string);
+                if (m.method === "OnConnected") return res();
+                if (m.id && pend.has(m.id)) { const x = pend.get(m.id)!; pend.delete(m.id); m.error ? x.rej(new Error(JSON.stringify(m.error))) : x.res(m.result); }
+            };
+            ws.onerror = () => rej(new Error("ws error (403 proxy ?)"));
+            setTimeout(() => rej(new Error("ws timeout")), 30000);
+        });
+        try {
+            await ready;
+            const open = await rpc("OpenDoc", { qDocName: p.app });
+            const doc = open.qReturn.qHandle;
+
+            // Sélection des codes du fournisseur sur le champ derrière "Article Code" (efficacité)
+            if (p.codes.length) {
+                try {
+                    const gd = await rpc("GetDimension", { qId: p.dim }, doc);
+                    const dh = gd.qReturn.qHandle;
+                    const dl = await rpc("GetLayout", {}, dh);
+                    const field = dl.qLayout?.qDim?.qFieldDefs?.[0];
+                    if (field) {
+                        const gf = await rpc("GetField", { qFieldName: field }, doc);
+                        const fh = gf.qReturn.qHandle;
+                        await rpc("SelectValues", { qFieldValues: p.codes.map((c) => ({ qText: c })), qToggleMode: false, qSoftLock: true }, fh);
+                    }
+                } catch { /* sélection best-effort */ }
+            }
+
+            const obj = await rpc("CreateSessionObject", { qProp: { qInfo: { qType: "cf-network" }, qHyperCubeDef: {
+                qDimensions: [{ qLibraryId: p.dim, qNullSuppression: true }],
+                qMeasures: [{ qLibraryId: p.mca }, { qLibraryId: p.mqte }, { qLibraryId: p.mnb }],
+                qInitialDataFetch: [], qSuppressMissing: true,
+            } } }, doc);
+            const oh = obj.qReturn.qHandle;
+            const layout = await rpc("GetLayout", {}, oh);
+            const size = layout.qLayout.qHyperCube.qSize.qcy as number;
+
+            const out: Array<Array<string | number>> = [];
+            const PAGE = 2500;
+            for (let top = 0; top < size; top += PAGE) {
+                const d = await rpc("GetHyperCubeData", { qPath: "/qHyperCubeDef", qPages: [{ qTop: top, qLeft: 0, qWidth: 4, qHeight: PAGE }] }, oh);
+                const matrix = d.qDataPages?.[0]?.qMatrix ?? [];
+                if (!matrix.length) break;
+                for (const r of matrix) out.push([String(r[0].qText ?? ""), Number(r[1].qNum) || 0, Number(r[2].qNum) || 0, Number(r[3].qNum) || 0]);
+                if (matrix.length < PAGE) break;
+            }
+            ws.close();
+            return { ok: true, rows: out, size };
+        } catch (e: any) {
+            try { ws.close(); } catch {}
+            return { ok: false, error: String(e?.message || e) };
+        }
+    })();
+}
+
+export async function fetchNetworkMetricsPlaywright(
+    codeCentraux?: string[],
+    cfg: QlikConfig = getQlikConfig(),
+): Promise<Map<string, NetworkMetric>> {
+    if (!cfg.appNetwork) throw new Error("[qlik-pw] QLIK_APP_NETWORK manquant");
+    if (!cfg.user || !cfg.password) throw new Error("[qlik-pw] identifiants Qlik manquants");
+
+    const xrf = Array.from({ length: 16 }, () => "abcdefghijklmnopqrstuvwxyz0123456789"[Math.floor(Math.random() * 36)]).join("");
+    const browser = await getBrowser();
+    const ctx = await browser.newContext({
+        httpCredentials: { username: cfg.user, password: cfg.password, origin: `https://${cfg.host}` },
+        ignoreHTTPSErrors: true,
+    });
+    try {
+        const page = await ctx.newPage();
+        await page.goto(`https://${cfg.host}/sense/app/${cfg.appNetwork}`, { waitUntil: "domcontentloaded", timeout: cfg.timeoutMs })
+            .catch(() => { /* la page peut ne pas finir de charger, le ws in-page suffit */ });
+
+        const result = (await page.evaluate(inPageExtract as never, {
+            app: cfg.appNetwork, dim: cfg.dimCodeArticleId, mca: cfg.measCaId, mqte: cfg.measQteId, mnb: cfg.measNbMagId,
+            codes: codeCentraux ?? [], xrf,
+        })) as InPageResult;
+
+        if (!result.ok) throw new Error(`[qlik-pw] extraction: ${result.error}`);
+
+        const wanted = codeCentraux ? new Set(codeCentraux) : null;
+        const out = new Map<string, NetworkMetric>();
+        for (const r of result.rows ?? []) {
+            const code = String(r[0]).trim();
+            if (!code) continue;
+            if (wanted && !wanted.has(code)) continue;
+            out.set(code, { codeCentrale: code, caReseau: Number(r[1]) || 0, qteReseau: Number(r[2]) || 0, nbMagasinsReseau: Number(r[3]) || 0 });
+        }
+        console.log(`[qlik-pw] ${out.size} produits réseau (cube size ${result.size})`);
+        return out;
+    } finally {
+        await ctx.close();
+    }
+}