From a9480a47b3081b3beb682b02afc000400a65986c Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Nov 2025 09:33:41 +0000 Subject: [PATCH] refactor: Remove Crawl4AI proxy config, rely on Browserless system - Removed proxy_config from BrowserConfig to avoid conflicts - Browserless service already has integrated proxy rotation - Will integrate with browserless_service later if needed - Focus on fixing CSS selectors first --- app/services/amazon_scraper.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/app/services/amazon_scraper.py b/app/services/amazon_scraper.py index cc70e1b..b64dc50 100644 --- a/app/services/amazon_scraper.py +++ b/app/services/amazon_scraper.py @@ -207,22 +207,15 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon user_agent = random.choice(AMAZON_USER_AGENTS) logger.debug(f"🎭 User-Agent: {user_agent[:50]}...") - # Select random proxy - proxy = get_random_proxy() - if proxy: - # Extract just the IP for logging (hide credentials) - proxy_parts = proxy.split('@') - proxy_server = proxy_parts[1] if len(proxy_parts) > 1 else proxy - logger.debug(f"🌐 Using proxy: {proxy_server}") - else: - logger.warning("⚠️ No proxy available - may face rate limiting") + # NOTE: Not using proxies here - Browserless service has its own proxy system + # We can integrate with browserless_service later if needed # Configure Crawl4AI browser with anti-detection browser_config = BrowserConfig( headless=True, verbose=False, user_agent=user_agent, - proxy_config=proxy, # Use proxy_config instead of deprecated proxy + # proxy_config removed - let Crawl4AI use default or integrate with Browserless later extra_args=[ "--disable-blink-features=AutomationControlled", # Disable automation detection "--disable-dev-shm-usage",