refactor: Remove Crawl4AI proxy config, rely on Browserless system

- Removed proxy_config from BrowserConfig to avoid conflicts
- Browserless service already has integrated proxy rotation
- Will integrate with browserless_service later if needed
- Focus on fixing CSS selectors first
This commit is contained in:
Claude committed 2025-11-30 09:33:41 +00:00
1 parent 55276149a4
commit a9480a47b3
1 file changed
+3 -10
+3 -10
View File
@@ -207,22 +207,15 @@ async def scrape_amazon_search(query: str, max_results: int = 20) -> list[Amazon
user_agent = random.choice(AMAZON_USER_AGENTS)
logger.debug(f"🎭 User-Agent: {user_agent[:50]}...")
# Select random proxy
proxy = get_random_proxy()
if proxy:
# Extract just the IP for logging (hide credentials)
proxy_parts = proxy.split('@')
proxy_server = proxy_parts[1] if len(proxy_parts) > 1 else proxy
logger.debug(f"🌐 Using proxy: {proxy_server}")
else:
logger.warning("⚠️ No proxy available - may face rate limiting")
# NOTE: Not using proxies here - Browserless service has its own proxy system
# We can integrate with browserless_service later if needed
# Configure Crawl4AI browser with anti-detection
browser_config = BrowserConfig(
headless=True,
verbose=False,
user_agent=user_agent,
proxy_config=proxy, # Use proxy_config instead of deprecated proxy
# proxy_config removed - let Crawl4AI use default or integrate with Browserless later
extra_args=[
"--disable-blink-features=AutomationControlled", # Disable automation detection
"--disable-dev-shm-usage",