From 038dc43b60856d34375e9056c5590267a7e1427a Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Sun, 30 Nov 2025 09:26:02 +0000
Subject: [PATCH] fix: Correct proxy configuration for Crawl4AI compatibility

- Changed proxy format from dict to string (http://user:pass@ip:port)
- Updated get_random_proxy() to return Crawl4AI-compatible format
- Replaced deprecated 'proxy' with 'proxy_config' in BrowserConfig
- Fixed test script to handle new proxy format
- Added credential hiding in proxy logging for security

Fixes AttributeError: 'dict' object has no attribute 'strip'
---
 app/services/amazon_scraper.py | 29 ++++++++++++++++++++---------
 test_amazon_scraper.py         |  5 ++++-
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/app/services/amazon_scraper.py b/app/services/amazon_scraper.py
index 03ccbac..69ddb8c 100644
--- a/app/services/amazon_scraper.py
+++ b/app/services/amazon_scraper.py
@@ -59,8 +59,13 @@ def get_realistic_headers(user_agent: str) -> dict:
     }
 
 
-def get_random_proxy() -> dict | None:
-    """Get a random proxy from the pool"""
+def get_random_proxy() -> str | None:
+    """
+    Get a random proxy from the pool in Crawl4AI format.
+
+    Returns:
+        Proxy string in format: http://username:password@ip:port
+    """
     if not AMAZON_PROXY_LIST_RAW:
         return None
 
@@ -68,11 +73,14 @@ def get_random_proxy() -> dict | None:
     parts = proxy_str.split(":")
 
     if len(parts) == 4:
-        return {
-            "server": f"http://{parts[0]}:{parts[1]}",
-            "username": parts[2],
-            "password": parts[3]
-        }
+        ip = parts[0]
+        port = parts[1]
+        username = parts[2]
+        password = parts[3]
+
+        # Format: http://username:password@ip:port
+        return f"http://{username}:{password}@{ip}:{port}"
+
     return None
 
 
@@ -202,7 +210,10 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon
     # Select random proxy
     proxy = get_random_proxy()
     if proxy:
-        logger.debug(f"🌐 Using proxy: {proxy['server']}")
+        # Extract just the IP for logging (hide credentials)
+        proxy_parts = proxy.split('@')
+        proxy_server = proxy_parts[1] if len(proxy_parts) > 1 else proxy
+        logger.debug(f"🌐 Using proxy: {proxy_server}")
     else:
         logger.warning("⚠️ No proxy available - may face rate limiting")
 
@@ -211,7 +222,7 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon
         headless=True,
         verbose=False,
         user_agent=user_agent,
-        proxy=proxy,
+        proxy_config=proxy,  # Use proxy_config instead of deprecated proxy
         extra_args=[
             "--disable-blink-features=AutomationControlled",  # Disable automation detection
             "--disable-dev-shm-usage",
diff --git a/test_amazon_scraper.py b/test_amazon_scraper.py
index d33b9f5..dd429e6 100755
--- a/test_amazon_scraper.py
+++ b/test_amazon_scraper.py
@@ -98,7 +98,10 @@ async def test_anti_detection():
     # Test proxy
     proxy = get_random_proxy()
     if proxy:
-        logger.info(f"✓ Proxy test: {proxy['server']}")
+        # Extract just the IP for logging (hide credentials)
+        proxy_parts = proxy.split('@')
+        proxy_server = proxy_parts[1] if len(proxy_parts) > 1 else proxy
+        logger.info(f"✓ Proxy test: {proxy_server}")
     else:
         logger.warning("⚠️ Pas de proxy configuré")