fix: Add stealth JavaScript injection to bypass Amazon bot detection

- Add STEALTH_JS constant with scripts to hide automation fingerprints
- Override navigator.webdriver, plugins, languages, permissions
- Fake WebGL vendor/renderer and hardware concurrency
- Inject stealth script via context.add_init_script() for Amazon pages
- Add bypass_csp=True to allow script injection
- Don't block images for Amazon to avoid detection triggers
- Apply stealth to both direct_search_service and scraper_service
This commit is contained in:
Claude committed 2025-11-24 08:07:49 +00:00
1 parent e62117f9b9
commit 3bae422c38
2 files changed
+222 -2

No files matched your search

+108
View File
@@ -109,6 +109,106 @@ async def _simulate_human_behavior(page):
USER_AGENT = USER_AGENT_POOL[0]
# Stealth JavaScript to inject before page load to hide automation
STEALTH_JS = """
// Override the navigator.webdriver property
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
configurable: true
});
// Override navigator.plugins to have some plugins
Object.defineProperty(navigator, 'plugins', {
get: () => {
const plugins = [
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
{ name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
];
plugins.item = (i) => plugins[i];
plugins.namedItem = (name) => plugins.find(p => p.name === name);
plugins.refresh = () => {};
return plugins;
},
configurable: true
});
// Override navigator.languages
Object.defineProperty(navigator, 'languages', {
get: () => ['fr-FR', 'fr', 'en-US', 'en'],
configurable: true
});
// Override permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
// Hide automation-related properties
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
// Override chrome object
window.chrome = {
runtime: {},
loadTimes: function() {},
csi: function() {},
app: {}
};
// Fake WebGL vendor and renderer
const getParameterProxyHandler = {
apply: function(target, ctx, args) {
if (args[0] === 37445) {
return 'Intel Inc.';
}
if (args[0] === 37446) {
return 'Intel Iris OpenGL Engine';
}
return Reflect.apply(target, ctx, args);
}
};
try {
const canvas = document.createElement('canvas');
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
if (gl) {
const getParameter = gl.getParameter.bind(gl);
gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
}
} catch(e) {}
// Override connection rtt (reduces fingerprinting)
Object.defineProperty(navigator, 'connection', {
get: () => ({
effectiveType: '4g',
rtt: 50,
downlink: 10,
saveData: false
}),
configurable: true
});
// Override hardware concurrency
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8,
configurable: true
});
// Override device memory
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8,
configurable: true
});
console.log('Stealth mode activated');
"""
# Chemin absolu pour les dumps de débogage (doit correspondre à celui dans debug.py)
DEBUG_DUMPS_DIR = "/app/debug_dumps"
@@ -695,8 +795,16 @@ async def _search_site_browserless( # noqa: PLR0912, PLR0915
locale="fr-FR",
timezone_id="Europe/Paris",
extra_http_headers=extra_headers,
# Additional stealth options
java_script_enabled=True,
bypass_csp=True, # Bypass Content Security Policy for script injection
)
# Inject stealth script before any page loads (especially for Amazon)
if is_amazon:
await context.add_init_script(STEALTH_JS)
logger.debug("Stealth JS injected for Amazon")
# For Amazon, don't block images - it may trigger detection
if not is_amazon:
await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
+114 -2
View File
@@ -41,6 +41,105 @@ AMAZON_BLOCK_INDICATORS = [
"automated access",
]
# Stealth JavaScript to inject before page load to hide automation
STEALTH_JS = """
// Override the navigator.webdriver property
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined,
configurable: true
});
// Override navigator.plugins to have some plugins
Object.defineProperty(navigator, 'plugins', {
get: () => {
const plugins = [
{ name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
{ name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
{ name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }
];
plugins.item = (i) => plugins[i];
plugins.namedItem = (name) => plugins.find(p => p.name === name);
plugins.refresh = () => {};
return plugins;
},
configurable: true
});
// Override navigator.languages
Object.defineProperty(navigator, 'languages', {
get: () => ['fr-FR', 'fr', 'en-US', 'en'],
configurable: true
});
// Override permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
// Hide automation-related properties
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Array;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Promise;
delete window.cdc_adoQpoasnfa76pfcZLmcfl_Symbol;
// Override chrome object
window.chrome = {
runtime: {},
loadTimes: function() {},
csi: function() {},
app: {}
};
// Fake WebGL vendor and renderer
const getParameterProxyHandler = {
apply: function(target, ctx, args) {
if (args[0] === 37445) {
return 'Intel Inc.';
}
if (args[0] === 37446) {
return 'Intel Iris OpenGL Engine';
}
return Reflect.apply(target, ctx, args);
}
};
try {
const canvas = document.createElement('canvas');
const gl = canvas.getContext('webgl') || canvas.getContext('experimental-webgl');
if (gl) {
const getParameter = gl.getParameter.bind(gl);
gl.getParameter = new Proxy(getParameter, getParameterProxyHandler);
}
} catch(e) {}
// Override connection rtt (reduces fingerprinting)
Object.defineProperty(navigator, 'connection', {
get: () => ({
effectiveType: '4g',
rtt: 50,
downlink: 10,
saveData: false
}),
configurable: true
});
// Override hardware concurrency
Object.defineProperty(navigator, 'hardwareConcurrency', {
get: () => 8,
configurable: true
});
// Override device memory
Object.defineProperty(navigator, 'deviceMemory', {
get: () => 8,
configurable: true
});
console.log('Stealth mode activated');
"""
def _get_random_user_agent() -> str:
"""Get a random User-Agent from the pool"""
@@ -269,10 +368,23 @@ class ScraperService:
locale="fr-FR",
timezone_id="Europe/Paris",
extra_http_headers=extra_headers,
# Additional stealth options
java_script_enabled=True,
bypass_csp=True, # Bypass Content Security Policy for script injection
)
# Stealth mode / Ad blocking attempts
await context.route("**/*", lambda route: route.continue_())
# Inject stealth script before any page loads (especially for Amazon)
if is_amazon:
await context.add_init_script(STEALTH_JS)
logger.debug("Stealth JS injected for Amazon product page")
# Stealth mode / Ad blocking attempts - don't block images for Amazon
if is_amazon:
# For Amazon, only block tracking - keep images to avoid detection
await context.route("**/analytics*", lambda route: route.abort())
await context.route("**/tracking*", lambda route: route.abort())
else:
await context.route("**/*", lambda route: route.continue_())
page = await context.new_page()