diff --git a/postgres/init.sql b/postgres/init.sql index 355b23e..464fceb 100644 --- a/postgres/init.sql +++ b/postgres/init.sql @@ -12,6 +12,13 @@ CREATE TABLE IF NOT EXISTS sync_log ( error_msg TEXT ); +-- Index photos lafoirfouille.fr (construit depuis les sitemaps) +CREATE TABLE IF NOT EXISTS ff_photo_index ( + photo_code TEXT PRIMARY KEY, + product_url TEXT NOT NULL, + indexed_at TIMESTAMP DEFAULT NOW() +); + -- ============================================================ -- Référentiel articles -- ============================================================ diff --git a/src/app.js b/src/app.js index 87d87af..e5d0bf6 100644 --- a/src/app.js +++ b/src/app.js @@ -13,6 +13,7 @@ const syncRouter = require('./routes/sync'); const rankingRouter = require('./routes/ranking'); const publicitesRouter = require('./routes/publicites'); const commandesAutoRouter = require('./routes/commandes-auto'); +const photosRouter = require('./routes/photos'); const app = express(); app.use(express.json()); @@ -40,6 +41,7 @@ app.use('/api/sync', syncRouter); app.use('/api/ranking', rankingRouter); app.use('/api/publicites', publicitesRouter); app.use('/api/commandes-auto', commandesAutoRouter); +app.use('/api/photos', photosRouter); // 404 app.use((req, res) => { diff --git a/src/routes/articles.js b/src/routes/articles.js index d238de1..3087b6a 100644 --- a/src/routes/articles.js +++ b/src/routes/articles.js @@ -283,31 +283,30 @@ const photoCache = new Map(); const FF_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'; -async function resolvePhotoUrl(code, size) { +async function resolvePhotoUrl(pool, code, size) { const cacheKey = `${code}:${size}`; const cached = photoCache.get(cacheKey); if (cached && cached.expires > Date.now()) return cached.imageUrl; - // 1. Cherche la page produit via search lafoirfouille.fr - const searchRes = await fetch(`https://www.lafoirfouille.fr/FR/search?q=${code}`, { - headers: { 'User-Agent': FF_UA, 'Accept-Language': 'fr-FR,fr;q=0.9' }, - redirect: 'follow', - }); - const searchHtml = await searchRes.text(); + // 1. Cherche l'URL produit dans l'index sitemap + const indexRow = await pool.query( + `SELECT product_url FROM ff_photo_index WHERE photo_code = $1`, + [code] + ); + if (!indexRow.rows.length) return null; - // Extrait URL produit du HTML (SSR partiel possible) - const urlMatch = searchHtml.match(new RegExp(`(/[^"']*?/p/[^"']*?-${code}\\.html)`)); - if (!urlMatch) return null; + const productPageUrl = indexRow.rows[0].product_url; // 2. Fetch page produit pour URL signée - const productRes = await fetch(`https://www.lafoirfouille.fr${urlMatch[1]}`, { + const productRes = await fetch(productPageUrl, { headers: { 'User-Agent': FF_UA, 'Accept-Language': 'fr-FR,fr;q=0.9' }, }); + if (!productRes.ok) return null; const productHtml = await productRes.text(); // Extrait URL CDN signée (pattern: medias/{code}-0-{size}?context=...) const imgPattern = new RegExp( - `https://prod-api\\.lafoirfouille\\.fr/medias/${code}-0-${size}\\?context=[A-Za-z0-9+/=_-]+` + `https://prod-api\\.lafoirfouille\\.fr/medias/${code}-0-${size}\\?context=[A-Za-z0-9+/=_%-]+` ); const imgMatch = productHtml.match(imgPattern); if (!imgMatch) return null; @@ -336,7 +335,8 @@ router.get('/:id/photo', async (req, res) => { if (!nomphoto) return res.status(404).json({ error: 'Pas de photo pour cet article' }); const code = nomphoto.replace(/\.[^.]+$/, ''); - const imageUrl = await resolvePhotoUrl(code, size); + if (!/^\d{9,13}$/.test(code)) return res.status(404).json({ error: 'Format photo non supporté', nomphoto }); + const imageUrl = await resolvePhotoUrl(pool, code, size); if (!imageUrl) return res.status(404).json({ error: 'Photo non trouvée sur lafoirfouille.fr', code }); diff --git a/src/routes/photos.js b/src/routes/photos.js new file mode 100644 index 0000000..c668299 --- /dev/null +++ b/src/routes/photos.js @@ -0,0 +1,87 @@ +const express = require('express'); +const router = express.Router(); +const { getPool } = require('../config/database'); + +const FF_SITEMAPS = [ + 'https://www.lafoirfouille.fr/sitemapProducts_0.xml', + 'https://www.lafoirfouille.fr/sitemapProducts_1.xml', + 'https://www.lafoirfouille.fr/sitemapProducts_2.xml', +]; +const FF_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'; +// Code produit lafoirfouille.fr : 9-13 chiffres en fin d'URL +const CODE_RE = /\/p\/[^/]+-(\d{9,13})\.html/g; + +let indexingInProgress = false; + +async function indexSitemaps(pool) { + let total = 0; + for (const sitemapUrl of FF_SITEMAPS) { + const res = await fetch(sitemapUrl, { headers: { 'User-Agent': FF_UA } }); + if (!res.ok) continue; + const xml = await res.text(); + + // Extrait toutes les URLs de produits + const urlMatches = [...xml.matchAll(/(https[^<]+)<\/loc>/g)]; + const rows = []; + for (const [, url] of urlMatches) { + const codeMatch = url.match(/\/p\/[^/]+-(\d{9,13})\.html/); + if (codeMatch) rows.push({ code: codeMatch[1], url }); + } + + if (!rows.length) continue; + + // Upsert par batch de 500 + for (let i = 0; i < rows.length; i += 500) { + const batch = rows.slice(i, i + 500); + const values = batch.map((r, j) => `($${j * 2 + 1}, $${j * 2 + 2})`).join(', '); + const params = batch.flatMap(r => [r.code, r.url]); + await pool.query( + `INSERT INTO ff_photo_index (photo_code, product_url, indexed_at) + VALUES ${values} + ON CONFLICT (photo_code) DO UPDATE + SET product_url = EXCLUDED.product_url, indexed_at = NOW()`, + params + ); + } + total += rows.length; + } + return total; +} + +// POST /api/photos/reindex — déclenche l'indexation des sitemaps +router.post('/reindex', async (req, res) => { + if (indexingInProgress) return res.status(409).json({ error: 'Indexation déjà en cours' }); + + const pool = getPool(); + indexingInProgress = true; + const start = Date.now(); + + try { + const total = await indexSitemaps(pool); + const elapsed = ((Date.now() - start) / 1000).toFixed(1); + res.json({ indexed: total, elapsed_s: parseFloat(elapsed) }); + } catch (err) { + res.status(500).json({ error: err.message }); + } finally { + indexingInProgress = false; + } +}); + +// GET /api/photos/status +router.get('/status', async (req, res) => { + try { + const pool = getPool(); + const result = await pool.query( + `SELECT COUNT(*) AS total, MAX(indexed_at) AS last_index FROM ff_photo_index` + ); + res.json({ + total: parseInt(result.rows[0].total), + last_index: result.rows[0].last_index, + indexing_in_progress: indexingInProgress, + }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +module.exports = router;