Merge branch 'antigravity' into main - Resolve conflicts

Resolved conflicts by keeping antigravity improvements:
- tracking_scraper_service.py: 47 popup selectors with RGPD/Amazon support
- main.py: No auto-init of TrackingScraperService (on-demand only)

This brings all popup handling improvements to main branch.

app/main.py

@@ -151,9 +151,8 @@ async def lifespan(app: FastAPI):
     logger.info("Initializing Improved Search Service...")
     await improved_search_service.initialize()
 
-    # Initialize Tracking Scraper Service
+    # Note: TrackingScraperService is initialized on-demand (not at startup)
-    logger.info("Initializing Tracking Scraper Service...")
+    # to avoid connection conflicts with other browser services
-    await TrackingScraperService.initialize()
 
     logger.info("Application started")
     yield
@@ -162,7 +161,7 @@ async def lifespan(app: FastAPI):
     stop_catalog_scheduler()
     await amazon_scraper_service.shutdown()
     await improved_search_service.shutdown()
-    await TrackingScraperService.shutdown()
+    # TrackingScraperService shutdown is handled on-demand
     logger.info("Application shutdown complete")
 
 

app/services/tracking_scraper_service.py

@@ -12,17 +12,56 @@ logger = logging.getLogger(__name__)
 BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000")
 
 POPUP_SELECTORS = [
+    # Amazon-specific popups
+    "#sp-cc-accept",  # Cookie banner Amazon
+    "#sp-cc-rejectall-link",
+    "button[data-action='a-popover-close']",
+    "[data-action='sp-cc-accept']",
+    "input[aria-labelledby='sp-cc-accept-label']",
+    "#nav-flyout-prime button",  # Prime flyout
+    
+    # French RGPD/Cookie consent platforms
+    "#axeptio_btn_acceptAll",  # Axeptio
+    "#axeptio_main_button",
+    "button#didomi-notice-agree-button",  # Didomi
+    ".didomi-popup-notice-button-accept",
+    "#onetrust-accept-btn-handler",  # OneTrust
+    "button.onetrust-close-btn-handler",
+    "#tarteaucitronPersonalize2",  # TarteAuCitron
+    "button#tarteaucitronAllAllowed",
+    
+    # Generic close buttons
     "button[aria-label='Close']",
     "button[aria-label='close']",
+    "button[aria-label='Fermer']",
+    "button[aria-label='fermer']",
     ".close-button",
     ".modal-close",
     "svg[data-name='Close']",
     "[class*='popup'] button",
     "[class*='modal'] button",
+    "[class*='overlay'] button",
+    
+    # Newsletter/subscription popups
     "button:has-text('No, thanks')",
     "button:has-text('No thanks')",
+    "button:has-text('Non merci')",
+    "button:has-text('Non, merci')",
     "a:has-text('No, thanks')",
+    "a:has-text('Non merci')",
+    
+    # Dialog/modal close buttons
     "div[role='dialog'] button[aria-label='Close']",
+    "div[role='dialog'] button[aria-label='Fermer']",
+    "[role='dialog'] .close",
+    
+    # Cookie banners (generic)
+    "#cookieConsent button",
+    ".cookie-consent button",
+    "button[id*='cookie'][id*='accept']",
+    "button[class*='cookie'][class*='accept']",
+    ".cc-dismiss",
+    ".cc-allow",
 ]
 
 
@@ -131,7 +170,7 @@ class TrackingScraperService:
 
             try:
                 await TrackingScraperService._navigate_and_wait(page, url, timeout)
-                await TrackingScraperService._handle_popups(page)
+                await TrackingScraperService._handle_popups(page, url)
 
                 if selector:
                     await TrackingScraperService._wait_for_selector(page, selector)
@@ -142,6 +181,11 @@ class TrackingScraperService:
                     await TrackingScraperService._smart_scroll(page, scroll_pixels)
 
                 page_text = await TrackingScraperService._extract_text(page, config.text_length)
+                
+                # Second popup handling pass right before screenshot to ensure clean capture
+                await TrackingScraperService._handle_popups(page, url)
+                await page.wait_for_timeout(1000)  # Final wait for any animations
+                
                 screenshot_path = await TrackingScraperService._take_screenshot(page, url, item_id)
 
                 return screenshot_path, page_text
@@ -192,21 +236,93 @@ class TrackingScraperService:
         await page.wait_for_timeout(2000)
 
     @staticmethod
-    async def _handle_popups(page: Page):
+    async def _handle_popups(page: Page, url: str = ""):
+        """
+        Close popups and cookie banners with retry logic and verification.
+        
+        Args:
+            page: Playwright page object
+            url: URL being scraped (for Amazon detection)
+        """
         logger.info("Attempting to close popups...")
+        is_amazon = "amazon" in url.lower() if url else False
+        
+        if is_amazon:
+            logger.info("🛒 Amazon detected - applying enhanced popup handling")
+        
+        closed_popups = []
+        
+        # First pass: Try to close all visible popups
         for popup_selector in POPUP_SELECTORS:
             try:
-                if await page.locator(popup_selector).count() > 0:
+                popup_count = await page.locator(popup_selector).count()
+                if popup_count > 0:
                     logger.info(f"Found popup close button: {popup_selector}")
-                    await page.locator(popup_selector).first.click(timeout=2000)
+                    await page.locator(popup_selector).first.click(timeout=3000)
-                    await page.wait_for_timeout(1000)
+                    closed_popups.append(popup_selector)
+                    await page.wait_for_timeout(1500)  # Increased wait for popup animation
+            except Exception as e:
+                logger.debug(f"Could not click {popup_selector}: {e}")
+                pass
+        
+        # Try Escape key multiple times (some sites need it)
+        for _ in range(2):
+            try:
+                await page.keyboard.press("Escape")
+                await page.wait_for_timeout(500)
             except Exception:
                 pass
         
-        try:
+        # Second pass: Verify and retry if any popups are still visible
-            await page.keyboard.press("Escape")
+        if closed_popups:
-        except Exception:
+            logger.info(f"Verifying {len(closed_popups)} closed popup(s)...")
-            pass
+            await page.wait_for_timeout(1000)
+            
+            for popup_selector in closed_popups:
+                try:
+                    if await page.locator(popup_selector).count() > 0:
+                        logger.warning(f"Popup reappeared, retrying: {popup_selector}")
+                        await page.locator(popup_selector).first.click(timeout=2000)
+                        await page.wait_for_timeout(1000)
+                except Exception:
+                    pass
+        
+        # Amazon-specific: Extra wait for JS-heavy popups
+        if is_amazon:
+            await page.wait_for_timeout(2000)
+            logger.info("✅ Amazon popup handling complete")
+        
+        logger.info(f"Popup handling complete ({len(closed_popups)} closed)")
+
+    @staticmethod
+    async def _verify_no_popups(page: Page) -> bool:
+        """
+        Verify that no modal/overlay elements are currently visible.
+        
+        Returns:
+            True if no popups are visible, False otherwise
+        """
+        overlay_selectors = [
+            "[role='dialog']",
+            ".modal[style*='display: block']",
+            ".popup[style*='display: block']",
+            "[class*='overlay'][style*='display: block']",
+            "[class*='modal-open']",
+        ]
+        
+        for selector in overlay_selectors:
+            try:
+                count = await page.locator(selector).count()
+                if count > 0:
+                    # Check if actually visible
+                    elem = page.locator(selector).first
+                    if await elem.is_visible():
+                        logger.warning(f"⚠️ Visible popup/overlay detected: {selector}")
+                        return False
+            except Exception:
+                pass
+        
+        return True
 
     @staticmethod
     async def _wait_for_selector(page: Page, selector: str):