diff --git a/Dockerfile b/Dockerfile index 2532557..e030d81 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,8 +37,8 @@ RUN chmod +x docker-entrypoint.sh # Copy built frontend static files COPY --from=frontend-build /app/frontend/dist /app/static -# Create screenshots directory -RUN mkdir -p screenshots +# Create screenshots and debug_dumps directories +RUN mkdir -p screenshots debug_dumps # Expose port EXPOSE 8555 diff --git a/app/alembic/versions/add_search_sites_columns.py b/app/alembic/versions/add_search_sites_columns.py new file mode 100644 index 0000000..b6d49be --- /dev/null +++ b/app/alembic/versions/add_search_sites_columns.py @@ -0,0 +1,69 @@ +"""Add missing columns to search_sites table + +Revision ID: add_search_sites_columns +Revises: add_notification_channels +Create Date: 2025-11-24 + +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy import inspect + +# revision identifiers, used by Alembic. +revision: str = "add_search_sites_columns" +down_revision: str | None = "add_notification_channels" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def column_exists(table_name: str, column_name: str) -> bool: + """Check if a column exists in a table.""" + bind = op.get_bind() + inspector = inspect(bind) + columns = [col["name"] for col in inspector.get_columns(table_name)] + return column_name in columns + + +def upgrade() -> None: + """Add debug_enabled, price_selector, search_url, product_link_selector columns to search_sites.""" + # Add debug_enabled column (only if not exists) + if not column_exists("search_sites", "debug_enabled"): + op.add_column( + "search_sites", + sa.Column("debug_enabled", sa.Boolean(), server_default="false", nullable=False), + ) + + # Add price_selector column (only if not exists) + if not column_exists("search_sites", "price_selector"): + op.add_column( + "search_sites", + sa.Column("price_selector", sa.String(), nullable=True), + ) + + # Add search_url column (only if not exists) + if not column_exists("search_sites", "search_url"): + op.add_column( + "search_sites", + sa.Column("search_url", sa.String(), nullable=True), + ) + + # Add product_link_selector column (only if not exists) + if not column_exists("search_sites", "product_link_selector"): + op.add_column( + "search_sites", + sa.Column("product_link_selector", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + """Remove added columns from search_sites.""" + if column_exists("search_sites", "product_link_selector"): + op.drop_column("search_sites", "product_link_selector") + if column_exists("search_sites", "search_url"): + op.drop_column("search_sites", "search_url") + if column_exists("search_sites", "price_selector"): + op.drop_column("search_sites", "price_selector") + if column_exists("search_sites", "debug_enabled"): + op.drop_column("search_sites", "debug_enabled") diff --git a/app/models.py b/app/models.py index cbee95e..d5d61e3 100644 --- a/app/models.py +++ b/app/models.py @@ -94,6 +94,7 @@ class SearchSite(Base): is_active: bool = Column(Boolean, default=True) # type: ignore priority: int = Column(Integer, default=0) # type: ignore # Ordre d'affichage requires_js: bool = Column(Boolean, default=False) # type: ignore # Force Browserless si True + debug_enabled: bool = Column(Boolean, default=False) # type: ignore # Activer le dump HTML pour ce site price_selector: str | None = Column(String, nullable=True) # type: ignore # Sélecteur CSS pour le prix search_url: str | None = Column(String, nullable=True) # type: ignore # URL de recherche avec {query} placeholder product_link_selector: str | None = Column(String, nullable=True) # type: ignore # Sélecteur CSS pour les liens produits diff --git a/app/routers/debug.py b/app/routers/debug.py index 07d694b..4714a02 100644 --- a/app/routers/debug.py +++ b/app/routers/debug.py @@ -18,7 +18,8 @@ router = APIRouter( logger = logging.getLogger(__name__) -DEBUG_DIR = "debug_dumps" +# Chemin absolu pour les dumps de débogage (doit correspondre à celui dans direct_search_service.py) +DEBUG_DIR = "/app/debug_dumps" class DebugFile(BaseModel): filename: str @@ -74,19 +75,82 @@ def delete_debug_dump(filename: str): """Delete a specific debug dump file""" if not os.path.exists(DEBUG_DIR): raise HTTPException(status_code=404, detail="Debug directory not found") - + filepath = os.path.join(DEBUG_DIR, filename) - + # Security check: prevent directory traversal if not os.path.abspath(filepath).startswith(os.path.abspath(DEBUG_DIR)): raise HTTPException(status_code=403, detail="Access denied") - + if not os.path.isfile(filepath): raise HTTPException(status_code=404, detail="File not found") - + try: os.remove(filepath) return {"message": "File deleted"} except Exception as e: logger.error(f"Error deleting debug dump {filename}: {e}") raise HTTPException(status_code=500, detail=str(e)) + + +@router.get("/dumps/site/{domain}", response_model=List[DebugFile]) +def list_debug_dumps_by_site(domain: str): + """List debug dump files for a specific site domain""" + if not os.path.exists(DEBUG_DIR): + return [] + + # Clean domain for comparison (remove www. if present) + clean_domain = domain.lower().strip() + if clean_domain.startswith("www."): + clean_domain = clean_domain[4:] + + files = [] + try: + for filename in os.listdir(DEBUG_DIR): + # Check if filename starts with the domain + filename_lower = filename.lower() + # Handle both formats: domain_query_timestamp.html and amazon.fr_query_timestamp.html + if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")): + filepath = os.path.join(DEBUG_DIR, filename) + if os.path.isfile(filepath): + stat = os.stat(filepath) + files.append(DebugFile( + filename=filename, + size=stat.st_size, + created_at=stat.st_ctime, + created_at_formatted=datetime.fromtimestamp(stat.st_ctime).strftime("%Y-%m-%d %H:%M:%S") + )) + + # Sort by creation time (newest first) + files.sort(key=lambda x: x.created_at, reverse=True) + return files + except Exception as e: + logger.error(f"Error listing debug dumps for site {domain}: {e}") + raise HTTPException(status_code=500, detail=str(e)) + + +@router.delete("/dumps/site/{domain}") +def delete_debug_dumps_by_site(domain: str): + """Delete all debug dump files for a specific site domain""" + if not os.path.exists(DEBUG_DIR): + return {"message": "No files found", "deleted": 0} + + # Clean domain for comparison + clean_domain = domain.lower().strip() + if clean_domain.startswith("www."): + clean_domain = clean_domain[4:] + + deleted_count = 0 + try: + for filename in os.listdir(DEBUG_DIR): + filename_lower = filename.lower() + if filename_lower.startswith(clean_domain) or filename_lower.startswith(clean_domain.replace(".", "_")): + filepath = os.path.join(DEBUG_DIR, filename) + if os.path.isfile(filepath): + os.remove(filepath) + deleted_count += 1 + + return {"message": f"{deleted_count} file(s) deleted", "deleted": deleted_count} + except Exception as e: + logger.error(f"Error deleting debug dumps for site {domain}: {e}") + raise HTTPException(status_code=500, detail=str(e)) diff --git a/app/routers/search_sites.py b/app/routers/search_sites.py index 9ae6bff..5df10bc 100644 --- a/app/routers/search_sites.py +++ b/app/routers/search_sites.py @@ -41,17 +41,17 @@ async def update_site( ): """ Met à jour un site de recherche. - Permet uniquement de modifier is_active et priority. + Permet uniquement de modifier is_active, priority et debug_enabled. """ # Limiter les champs modifiables - allowed_fields = {"is_active", "priority"} + allowed_fields = {"is_active", "priority", "debug_enabled"} update_data = site.model_dump(exclude_unset=True) filtered_data = {k: v for k, v in update_data.items() if k in allowed_fields} if not filtered_data: raise HTTPException( status_code=400, - detail="Seuls les champs 'is_active' et 'priority' peuvent être modifiés" + detail="Seuls les champs 'is_active', 'priority' et 'debug_enabled' peuvent être modifiés" ) updated = search_service.update_site(db, site_id, filtered_data) diff --git a/app/schemas.py b/app/schemas.py index 2f82e75..e34886c 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -53,6 +53,7 @@ class SearchSiteCreate(BaseModel): is_active: bool = True priority: int = 0 requires_js: bool = False + debug_enabled: bool = False # Activer le dump HTML pour ce site price_selector: str | None = None search_url: str | None = None # URL avec {query} placeholder, ex: https://amazon.fr/s?k={query} product_link_selector: str | None = None # Sélecteur CSS pour les liens produits @@ -66,6 +67,7 @@ class SearchSiteUpdate(BaseModel): is_active: bool | None = None priority: int | None = None requires_js: bool | None = None + debug_enabled: bool | None = None # Activer le dump HTML pour ce site price_selector: str | None = None search_url: str | None = None product_link_selector: str | None = None @@ -80,6 +82,7 @@ class SearchSiteResponse(BaseModel): is_active: bool priority: int requires_js: bool + debug_enabled: bool = False # Activer le dump HTML pour ce site price_selector: str | None = None search_url: str | None = None product_link_selector: str | None = None diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py index ffa7b24..b36551b 100644 --- a/app/services/scraper_service.py +++ b/app/services/scraper_service.py @@ -1,6 +1,7 @@ import asyncio import logging import os +import random import re from urllib.parse import urlparse, urlunparse @@ -11,10 +12,318 @@ logger = logging.getLogger(__name__) BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000") +# Proxy configuration for Amazon (optional) +# Format: "http://user:pass@host:port" or "http://host:port" +AMAZON_PROXY_URL = os.getenv("AMAZON_PROXY_URL", "") +AMAZON_PROXY_LIST = [p.strip() for p in os.getenv("AMAZON_PROXY_LIST", "").split(",") if p.strip()] + +def _get_amazon_proxy() -> str | None: + """Get a proxy for Amazon requests""" + if AMAZON_PROXY_LIST: + return random.choice(AMAZON_PROXY_LIST) + if AMAZON_PROXY_URL: + return AMAZON_PROXY_URL + return None + # Nombre de tentatives pour le scraping MAX_RETRIES = 2 RETRY_DELAY = 3 # secondes +# Amazon-specific configuration +AMAZON_MAX_RETRIES = 4 +AMAZON_BASE_DELAY = 3.0 + +# Pool of realistic User-Agents for rotation +USER_AGENT_POOL = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", +] + +# Amazon blocking indicators +AMAZON_BLOCK_INDICATORS = [ + "Toutes nos excuses", + "Sorry, something went wrong", + "api-services-support@amazon.com", + "Enter the characters you see below", + "Saisissez les caractères", + "Service Unavailable", + "robot check", + "automated access", +] + +# Stealth JavaScript to inject before page load to hide automation +STEALTH_JS = """ +// ============================================ +// COMPREHENSIVE STEALTH MODE FOR AMAZON +// ============================================ + +// 1. Override navigator.webdriver in multiple ways +Object.defineProperty(navigator, 'webdriver', { + get: () => false, + configurable: true +}); + +// Also delete from navigator prototype +delete Object.getPrototypeOf(navigator).webdriver; + +// Override the getAttribute method to hide webdriver attribute +const originalGetAttribute = Element.prototype.getAttribute; +Element.prototype.getAttribute = function(name) { + if (name === 'webdriver') return null; + return originalGetAttribute.call(this, name); +}; + +// 2. Override navigator.plugins with realistic plugins +Object.defineProperty(navigator, 'plugins', { + get: () => { + const plugins = [ + { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 }, + { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 }, + { name: 'Native Client', filename: 'internal-nacl-plugin', description: '', length: 2 }, + { name: 'Chromium PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format', length: 1 }, + { name: 'Chromium PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '', length: 1 } + ]; + plugins.item = (i) => plugins[i] || null; + plugins.namedItem = (name) => plugins.find(p => p.name === name) || null; + plugins.refresh = () => {}; + plugins.length = plugins.length; + return plugins; + }, + configurable: true +}); + +// 3. Override navigator.mimeTypes +Object.defineProperty(navigator, 'mimeTypes', { + get: () => { + const mimeTypes = [ + { type: 'application/pdf', suffixes: 'pdf', description: 'Portable Document Format' }, + { type: 'text/pdf', suffixes: 'pdf', description: 'Portable Document Format' } + ]; + mimeTypes.item = (i) => mimeTypes[i] || null; + mimeTypes.namedItem = (name) => mimeTypes.find(m => m.type === name) || null; + mimeTypes.length = mimeTypes.length; + return mimeTypes; + }, + configurable: true +}); + +// 4. Override navigator.languages +Object.defineProperty(navigator, 'languages', { + get: () => ['fr-FR', 'fr', 'en-US', 'en'], + configurable: true +}); + +// 5. Override permissions API +if (window.navigator.permissions) { + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => { + if (parameters.name === 'notifications') { + return Promise.resolve({ state: Notification.permission, onchange: null }); + } + return originalQuery.call(window.navigator.permissions, parameters); + }; +} + +// 6. Hide automation-related properties (Chromium DevTools Protocol) +const propsToDelete = [ + 'cdc_adoQpoasnfa76pfcZLmcfl_Array', + 'cdc_adoQpoasnfa76pfcZLmcfl_Promise', + 'cdc_adoQpoasnfa76pfcZLmcfl_Symbol', + '__webdriver_evaluate', + '__selenium_evaluate', + '__webdriver_script_function', + '__webdriver_script_func', + '__webdriver_script_fn', + '__fxdriver_evaluate', + '__driver_unwrapped', + '__webdriver_unwrapped', + '__driver_evaluate', + '__selenium_unwrapped', + '__fxdriver_unwrapped', + '_Selenium_IDE_Recorder', + '_selenium', + 'calledSelenium', + '$chrome_asyncScriptInfo', + '$cdc_asdjflasutopfhvcZLmcfl_', + '$wdc_' +]; +propsToDelete.forEach(prop => { + try { delete window[prop]; } catch(e) {} +}); + +// 7. Override chrome object with realistic properties +window.chrome = { + app: { + isInstalled: false, + InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, + RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' } + }, + runtime: { + OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' }, + OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' }, + PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, + PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, + PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' }, + RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' }, + connect: function() { return { onDisconnect: { addListener: function() {} }, onMessage: { addListener: function() {} }, postMessage: function() {} }; }, + sendMessage: function() {} + }, + csi: function() { return {}; }, + loadTimes: function() { return { requestTime: Date.now() / 1000, startLoadTime: Date.now() / 1000, firstPaintAfterLoadTime: 0, firstPaintTime: Date.now() / 1000, navigationType: 'navigate' }; } +}; + +// 8. WebGL fingerprinting protection +const getParameterProxyHandler = { + apply: function(target, ctx, args) { + if (args[0] === 37445) return 'Intel Inc.'; + if (args[0] === 37446) return 'Intel Iris OpenGL Engine'; + if (args[0] === 7937) return 'WebKit'; + if (args[0] === 7936) return 'WebKit WebGL'; + return Reflect.apply(target, ctx, args); + } +}; +try { + const canvas = document.createElement('canvas'); + const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl'); + if (gl) { + const getParameter = gl.getParameter.bind(gl); + gl.getParameter = new Proxy(getParameter, getParameterProxyHandler); + } + const gl2 = canvas.getContext('webgl2'); + if (gl2) { + const getParameter2 = gl2.getParameter.bind(gl2); + gl2.getParameter = new Proxy(getParameter2, getParameterProxyHandler); + } +} catch(e) {} + +// 9. Canvas fingerprinting protection +const originalToDataURL = HTMLCanvasElement.prototype.toDataURL; +HTMLCanvasElement.prototype.toDataURL = function(type) { + if (type === 'image/png' && this.width === 220 && this.height === 30) { + const context = this.getContext('2d'); + if (context) { + const imageData = context.getImageData(0, 0, this.width, this.height); + for (let i = 0; i < imageData.data.length; i += 4) { + imageData.data[i] = imageData.data[i] ^ (Math.random() * 2); + } + context.putImageData(imageData, 0, 0); + } + } + return originalToDataURL.apply(this, arguments); +}; + +// 10. AudioContext fingerprinting protection +if (window.AudioContext || window.webkitAudioContext) { + const AudioContext = window.AudioContext || window.webkitAudioContext; + const originalCreateOscillator = AudioContext.prototype.createOscillator; + AudioContext.prototype.createOscillator = function() { + const oscillator = originalCreateOscillator.apply(this, arguments); + oscillator.frequency.value = oscillator.frequency.value + (Math.random() * 0.0001); + return oscillator; + }; +} + +// 11. Override connection properties +Object.defineProperty(navigator, 'connection', { + get: () => ({ + effectiveType: '4g', + rtt: 50 + Math.floor(Math.random() * 50), + downlink: 10 + Math.random() * 5, + saveData: false + }), + configurable: true +}); + +// 12. Override hardware properties +Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8, + configurable: true +}); + +Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8, + configurable: true +}); + +// 13. Override screen properties +Object.defineProperty(screen, 'colorDepth', { + get: () => 24, + configurable: true +}); + +Object.defineProperty(screen, 'pixelDepth', { + get: () => 24, + configurable: true +}); + +// 14. Spoof Notification API +if (window.Notification) { + Object.defineProperty(Notification, 'permission', { + get: () => 'default', + configurable: true + }); +} + +// 15. Override iframe contentWindow checks +const originalContentWindow = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow'); +Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', { + get: function() { + const win = originalContentWindow.get.call(this); + if (win) { + try { + Object.defineProperty(win.navigator, 'webdriver', { + get: () => false, + configurable: true + }); + } catch(e) {} + } + return win; + } +}); + +// 16. Mock Battery API +if (navigator.getBattery) { + navigator.getBattery = () => Promise.resolve({ + charging: true, + chargingTime: 0, + dischargingTime: Infinity, + level: 1, + addEventListener: () => {}, + removeEventListener: () => {} + }); +} + +// 17. Hide Playwright/Puppeteer specific objects +delete window.__playwright; +delete window.__pw_manual; +delete window.__PW_inspect; + +console.log('Advanced stealth mode activated'); +""" + + +def _get_random_user_agent() -> str: + """Get a random User-Agent from the pool""" + return random.choice(USER_AGENT_POOL) + + +def _is_amazon_url(url: str) -> bool: + """Check if URL is an Amazon URL""" + return "amazon" in url.lower() + + +def _is_amazon_blocked(html_content: str) -> bool: + """Check if Amazon has blocked the request""" + html_lower = html_content.lower() + for indicator in AMAZON_BLOCK_INDICATORS: + if indicator.lower() in html_lower: + return True + return False + def simplify_url(url: str) -> str: """ @@ -117,12 +426,23 @@ class ScraperService: logger.warning(f"Invalid timeout value: {timeout}, using default 90000") timeout = 90000 + # Amazon needs more retries with longer delays + is_amazon = _is_amazon_url(url) + max_retries = AMAZON_MAX_RETRIES if is_amazon else MAX_RETRIES + base_delay = AMAZON_BASE_DELAY if is_amazon else RETRY_DELAY + # Retries avec backoff last_error = None - for attempt in range(MAX_RETRIES + 1): + for attempt in range(max_retries + 1): if attempt > 0: - logger.info(f"Tentative {attempt + 1}/{MAX_RETRIES + 1} pour {url}") - await asyncio.sleep(RETRY_DELAY * attempt) + # Exponential backoff with jitter for Amazon + if is_amazon: + jitter = random.uniform(0.5, 1.5) + delay = min(base_delay * (2 ** attempt) * jitter, 30.0) + else: + delay = base_delay * attempt + logger.info(f"Tentative {attempt + 1}/{max_retries + 1} pour {url} (délai: {delay:.1f}s)") + await asyncio.sleep(delay) result = await ScraperService._do_scrape( url=url, @@ -133,6 +453,7 @@ class ScraperService: text_length=text_length, timeout=timeout, browser=browser, + is_amazon=is_amazon, ) if result[0] is not None: # Screenshot réussi @@ -154,14 +475,15 @@ class ScraperService: text_length: int = 0, timeout: int = 90000, browser=None, + is_amazon: bool = False, ) -> tuple[str | None, str, bool]: """Exécute le scraping réel (appelé par scrape_item avec retries).""" - + # Si un navigateur est fourni, on l'utilise directement sans async_playwright context manager # sinon on crée tout de zéro playwright_manager = None local_browser = None - + try: if browser: # Utiliser le navigateur partagé @@ -180,48 +502,105 @@ class ScraperService: context = None page = None is_available = True # Par défaut, le produit est disponible - - try: - context = await current_browser.new_context( - viewport={"width": 1920, "height": 1080}, - user_agent=( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/120.0.0.0 Safari/537.36" - ), - locale="fr-FR", - timezone_id="Europe/Paris", - # Options supplémentaires pour éviter la détection - extra_http_headers={ - "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", - } - ) - # Stealth mode / Ad blocking attempts - await context.route("**/*", lambda route: route.continue_()) + # Use random User-Agent for anti-detection (especially for Amazon) + current_user_agent = _get_random_user_agent() + + # Enhanced headers for stealth + extra_headers = { + "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "Accept-Encoding": "gzip, deflate, br", + "Cache-Control": "max-age=0", + "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": '"Windows"', + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", + } + + # Add referer for Amazon + if is_amazon: + extra_headers["Referer"] = "https://www.google.fr/" + + try: + # Prepare context options + context_options = { + "viewport": {"width": 1920, "height": 1080}, + "user_agent": current_user_agent, + "locale": "fr-FR", + "timezone_id": "Europe/Paris", + "extra_http_headers": extra_headers, + "java_script_enabled": True, + "bypass_csp": True, + } + + # Add proxy for Amazon if configured + if is_amazon: + proxy_url = _get_amazon_proxy() + if proxy_url: + logger.info(f"Using proxy for Amazon: {proxy_url.split('@')[-1] if '@' in proxy_url else proxy_url}") + context_options["proxy"] = {"server": proxy_url} + + context = await current_browser.new_context(**context_options) + + # Inject stealth script before any page loads (especially for Amazon) + if is_amazon: + await context.add_init_script(STEALTH_JS) + logger.debug("Stealth JS injected for Amazon product page") + + # Stealth mode / Ad blocking attempts - don't block images for Amazon + if is_amazon: + # For Amazon, only block tracking - keep images to avoid detection + await context.route("**/analytics*", lambda route: route.abort()) + await context.route("**/tracking*", lambda route: route.abort()) + else: + await context.route("**/*", lambda route: route.continue_()) page = await context.new_page() + # Add random delay before navigation for Amazon + if is_amazon: + await asyncio.sleep(random.uniform(1.0, 3.0)) + + # Warmup: Visit home page first + try: + logger.info("Amazon Warmup: Visiting home page...") + await page.goto("https://www.amazon.fr/", wait_until="domcontentloaded", timeout=20000) + await asyncio.sleep(random.uniform(2.0, 4.0)) + except Exception as e: + logger.warning(f"Amazon warmup failed: {e}") + logger.info(f"Navigating to {url} (Timeout: {timeout}ms)") response = None + amazon_blocked = False try: # First wait for domcontentloaded - this is the minimum we need response = await page.goto(url, wait_until="domcontentloaded", timeout=timeout) - + # Check HTTP status code if response: status = response.status logger.info(f"HTTP Status: {status}") - - # Detect unavailable products by HTTP status - if status in [404, 410, 451, 503]: + + # For Amazon, 503 is usually a block, not product unavailability + if is_amazon and status == 503: + logger.warning(f"Amazon returned 503 - likely blocking") + amazon_blocked = True + # Detect unavailable products by HTTP status (non-Amazon) + elif status in [404, 410, 451]: + logger.warning(f"Product unavailable - HTTP {status}") + is_available = False + elif status == 503 and not is_amazon: logger.warning(f"Product unavailable - HTTP {status}") is_available = False elif status >= 400: logger.warning(f"HTTP error {status}, marking as potentially unavailable") is_available = False - + logger.info(f"Page loaded (domcontentloaded): {url}") # Then try to wait for networkidle, but don't fail if it times out @@ -239,8 +618,24 @@ class ScraperService: # Try to take screenshot anyway if page partially loaded pass - # Wait a bit for dynamic content if needed - await page.wait_for_timeout(2000) + # Wait a bit for dynamic content if needed - longer for Amazon + wait_time = random.uniform(2.0, 4.0) if is_amazon else 2.0 + await page.wait_for_timeout(int(wait_time * 1000)) + + # Check for Amazon blocking in page content + if is_amazon: + try: + page_html = await page.content() + if _is_amazon_blocked(page_html): + logger.warning(f"Amazon blocking detected in page content for {url}") + amazon_blocked = True + except Exception as e: + logger.debug(f"Could not check Amazon blocking: {e}") + + # If Amazon blocked us, return None to trigger retry + if amazon_blocked: + logger.warning(f"Amazon blocked - will retry with different User-Agent") + return None, "", True # Return None screenshot to trigger retry # Try to close common popups and cookie banners logger.info("Attempting to close popups and cookie banners...") diff --git a/app/services/search_service.py b/app/services/search_service.py index 7665b38..c2a5934 100644 --- a/app/services/search_service.py +++ b/app/services/search_service.py @@ -76,6 +76,7 @@ async def search_products( "search_url": site.search_url, "product_link_selector": site.product_link_selector, "requires_js": site.requires_js, + "debug_enabled": site.debug_enabled, } for site in sites ] diff --git a/debug_amazon.py b/debug_amazon.py new file mode 100644 index 0000000..2497fbf --- /dev/null +++ b/debug_amazon.py @@ -0,0 +1,60 @@ +import asyncio +import logging +import os +from playwright.async_api import async_playwright + +# Configure logging +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000") + +async def debug_amazon(): + logger.info("Starting Amazon Debug Script") + + async with async_playwright() as p: + try: + logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}") + browser = await p.chromium.connect_over_cdp(BROWSERLESS_URL) + + # Use a very standard, recent User-Agent + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" + + context = await browser.new_context( + user_agent=user_agent, + viewport={"width": 1920, "height": 1080}, + locale="fr-FR", + timezone_id="Europe/Paris" + ) + + page = await context.new_page() + + url = "https://www.amazon.fr/s?k=iphone" + logger.info(f"Navigating to {url}") + + response = await page.goto(url, wait_until="domcontentloaded", timeout=30000) + + if response: + status = response.status + logger.info(f"Response Status: {status}") + + content = await page.content() + if "api-services-support@amazon.com" in content or "Toutes nos excuses" in content: + logger.error("BLOCK DETECTED: Found blocking message in content") + else: + logger.info("No obvious blocking message found") + + # Save screenshot + await page.screenshot(path="debug_amazon_screenshot.png") + logger.info("Screenshot saved to debug_amazon_screenshot.png") + + else: + logger.error("No response received") + + await browser.close() + + except Exception as e: + logger.error(f"An error occurred: {e}") + +if __name__ == "__main__": + asyncio.run(debug_amazon()) diff --git a/frontend/src/pages/Admin.jsx b/frontend/src/pages/Admin.jsx index 2ffe8d8..1d7af6f 100644 --- a/frontend/src/pages/Admin.jsx +++ b/frontend/src/pages/Admin.jsx @@ -437,13 +437,43 @@ export default function Admin() { // Site functions const toggleSiteActive = async (site) => { try { - await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }); + await axios.put(`${API_URL}/search-sites/${site.id}`, { is_active: !site.is_active }, { headers: getAuthHeaders() }); fetchAll(); } catch (error) { toast.error('Erreur lors de la mise à jour'); } }; + const toggleSiteDebug = async (site) => { + try { + await axios.put(`${API_URL}/search-sites/${site.id}`, { debug_enabled: !site.debug_enabled }, { headers: getAuthHeaders() }); + toast.success(site.debug_enabled ? 'Debug désactivé' : 'Debug activé'); + fetchAll(); + } catch (error) { + toast.error('Erreur lors de la mise à jour du debug'); + } + }; + + const fetchSiteDebugFiles = async (domain) => { + try { + const response = await axios.get(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() }); + return response.data; + } catch (error) { + console.error('Error fetching debug files:', error); + return []; + } + }; + + const deleteSiteDebugFiles = async (domain) => { + if (!confirm(`Supprimer tous les fichiers de debug pour ${domain} ?`)) return; + try { + const response = await axios.delete(`${API_URL}/debug/dumps/site/${domain}`, { headers: getAuthHeaders() }); + toast.success(response.data.message); + } catch (error) { + toast.error('Erreur lors de la suppression'); + } + }; + const resetSitesToDefaults = async () => { if (!confirm('Êtes-vous sûr de vouloir réinitialiser tous les sites ?')) return; setResettingSites(true); @@ -862,11 +892,35 @@ export default function Admin() { toggleSiteActive(site)} className="ml-2" /> - {site.category && ( - - {site.category} - - )} +
+ {site.category && ( + + {site.category} + + )} +
+ + {site.debug_enabled && ( + + )} +
+
))} @@ -1018,6 +1072,54 @@ export default function Admin() { )} + + {/* Debug Tab */} + {activeTab === 'debug' && ( + + +
+
+ Debug HTML Dumps + Fichiers HTML sauvegardés lors des recherches sans résultats. +
+ +
+ + + {loadingDebugFiles ? ( +
Chargement...
+ ) : debugFiles.length === 0 ? ( +
+ Aucun fichier de debug disponible. +
+ ) : ( +
+ {debugFiles.map(file => ( +
+
+

{file.filename}

+

+ {(file.size / 1024).toFixed(1)} KB • {file.created_at_formatted} +

+
+
+ + +
+
+ ))} +
+ )} + + + )} ); }