From 3bae422c38e935453f2217cd27c1202580d515cc Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 24 Nov 2025 08:07:49 +0000 Subject: [PATCH] fix: Add stealth JavaScript injection to bypass Amazon bot detection - Add STEALTH_JS constant with scripts to hide automation fingerprints - Override navigator.webdriver, plugins, languages, permissions - Fake WebGL vendor/renderer and hardware concurrency - Inject stealth script via context.add_init_script() for Amazon pages - Add bypass_csp=True to allow script injection - Don't block images for Amazon to avoid detection triggers - Apply stealth to both direct_search_service and scraper_service --- app/services/direct_search_service.py | 108 ++++++++++++++++++++++++ app/services/scraper_service.py | 116 +++++++++++++++++++++++++- 2 files changed, 222 insertions(+), 2 deletions(-) diff --git a/app/services/direct_search_service.py b/app/services/direct_search_service.py index f937bc8..5b0a350 100644 --- a/app/services/direct_search_service.py +++ b/app/services/direct_search_service.py @@ -109,6 +109,106 @@ async def _simulate_human_behavior(page): USER_AGENT = USER_AGENT_POOL[0] +# Stealth JavaScript to inject before page load to hide automation +STEALTH_JS = """ +// Override the navigator.webdriver property +Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + configurable: true +}); + +// Override navigator.plugins to have some plugins +Object.defineProperty(navigator, 'plugins', { + get: () => { + const plugins = [ + { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, + { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, + { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' } + ]; + plugins.item = (i) => plugins[i]; + plugins.namedItem = (name) => plugins.find(p => p.name === name); + plugins.refresh = () => {}; + return plugins; + }, + configurable: true +}); + +// Override navigator.languages +Object.defineProperty(navigator, 'languages', { + get: () => ['fr-FR', 'fr', 'en-US', 'en'], + configurable: true +}); + +// Override permissions +const originalQuery = window.navigator.permissions.query; +window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters) +); + +// Hide automation-related properties +delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array; +delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise; +delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol; + +// Override chrome object +window.chrome = { + runtime: {}, + loadTimes: function() {}, + csi: function() {}, + app: {} +}; + +// Fake WebGL vendor and renderer +const getParameterProxyHandler = { + apply: function(target, ctx, args) { + if (args[0] === 37445) { + return 'Intel Inc.'; + } + if (args[0] === 37446) { + return 'Intel Iris OpenGL Engine'; + } + return Reflect.apply(target, ctx, args); + } +}; + +try { + const canvas = document.createElement('canvas'); + const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl'); + if (gl) { + const getParameter = gl.getParameter.bind(gl); + gl.getParameter = new Proxy(getParameter, getParameterProxyHandler); + } +} catch(e) {} + +// Override connection rtt (reduces fingerprinting) +Object.defineProperty(navigator, 'connection', { + get: () => ({ + effectiveType: '4g', + rtt: 50, + downlink: 10, + saveData: false + }), + configurable: true +}); + +// Override hardware concurrency +Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8, + configurable: true +}); + +// Override device memory +Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8, + configurable: true +}); + +console.log('Stealth mode activated'); +""" + + # Chemin absolu pour les dumps de débogage (doit correspondre à celui dans debug.py) DEBUG_DUMPS_DIR = "/app/debug_dumps" @@ -695,8 +795,16 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915 locale="fr-FR", timezone_id="Europe/Paris", extra_http_headers=extra_headers, + # Additional stealth options + java_script_enabled=True, + bypass_csp=True, # Bypass Content Security Policy for script injection ) + # Inject stealth script before any page loads (especially for Amazon) + if is_amazon: + await context.add_init_script(STEALTH_JS) + logger.debug("Stealth JS injected for Amazon") + # For Amazon, don't block images - it may trigger detection if not is_amazon: await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort()) diff --git a/app/services/scraper_service.py b/app/services/scraper_service.py index be0c931..e0e1d18 100644 --- a/app/services/scraper_service.py +++ b/app/services/scraper_service.py @@ -41,6 +41,105 @@ AMAZON_BLOCK_INDICATORS = [ "automated access", ] +# Stealth JavaScript to inject before page load to hide automation +STEALTH_JS = """ +// Override the navigator.webdriver property +Object.defineProperty(navigator, 'webdriver', { + get: () => undefined, + configurable: true +}); + +// Override navigator.plugins to have some plugins +Object.defineProperty(navigator, 'plugins', { + get: () => { + const plugins = [ + { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, + { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, + { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' } + ]; + plugins.item = (i) => plugins[i]; + plugins.namedItem = (name) => plugins.find(p => p.name === name); + plugins.refresh = () => {}; + return plugins; + }, + configurable: true +}); + +// Override navigator.languages +Object.defineProperty(navigator, 'languages', { + get: () => ['fr-FR', 'fr', 'en-US', 'en'], + configurable: true +}); + +// Override permissions +const originalQuery = window.navigator.permissions.query; +window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters) +); + +// Hide automation-related properties +delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array; +delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise; +delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol; + +// Override chrome object +window.chrome = { + runtime: {}, + loadTimes: function() {}, + csi: function() {}, + app: {} +}; + +// Fake WebGL vendor and renderer +const getParameterProxyHandler = { + apply: function(target, ctx, args) { + if (args[0] === 37445) { + return 'Intel Inc.'; + } + if (args[0] === 37446) { + return 'Intel Iris OpenGL Engine'; + } + return Reflect.apply(target, ctx, args); + } +}; + +try { + const canvas = document.createElement('canvas'); + const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl'); + if (gl) { + const getParameter = gl.getParameter.bind(gl); + gl.getParameter = new Proxy(getParameter, getParameterProxyHandler); + } +} catch(e) {} + +// Override connection rtt (reduces fingerprinting) +Object.defineProperty(navigator, 'connection', { + get: () => ({ + effectiveType: '4g', + rtt: 50, + downlink: 10, + saveData: false + }), + configurable: true +}); + +// Override hardware concurrency +Object.defineProperty(navigator, 'hardwareConcurrency', { + get: () => 8, + configurable: true +}); + +// Override device memory +Object.defineProperty(navigator, 'deviceMemory', { + get: () => 8, + configurable: true +}); + +console.log('Stealth mode activated'); +""" + def _get_random_user_agent() -> str: """Get a random User-Agent from the pool""" @@ -269,10 +368,23 @@ class ScraperService: locale="fr-FR", timezone_id="Europe/Paris", extra_http_headers=extra_headers, + # Additional stealth options + java_script_enabled=True, + bypass_csp=True, # Bypass Content Security Policy for script injection ) - # Stealth mode / Ad blocking attempts - await context.route("**/*", lambda route: route.continue_()) + # Inject stealth script before any page loads (especially for Amazon) + if is_amazon: + await context.add_init_script(STEALTH_JS) + logger.debug("Stealth JS injected for Amazon product page") + + # Stealth mode / Ad blocking attempts - don't block images for Amazon + if is_amazon: + # For Amazon, only block tracking - keep images to avoid detection + await context.route("**/analytics*", lambda route: route.abort()) + await context.route("**/tracking*", lambda route: route.abort()) + else: + await context.route("**/*", lambda route: route.continue_()) page = await context.new_page()