feat: add Playwright-based scraping service for tracking items.

app/services/tracking_scraper_service.py

@@ -228,20 +228,75 @@ class ScraperService:
 
     @staticmethod
     async def _handle_popups(page: Page):
-        logger.info("Attempting to close popups...")
+        """Aggressive multi-pass popup and overlay removal"""
-        for popup_selector in POPUP_SELECTORS:
+        logger.info("Starting aggressive popup removal...")
+        
+        # 1. Multi-pass clicking (some popups appear after others are closed)
+        for i in range(2):
+            logger.debug(f"Popup removal pass {i+1}")
+            for selector in POPUP_SELECTORS:
+                try:
+                    locators = page.locator(selector)
+                    count = await locators.count()
+                    if count > 0:
+                        for j in range(count):
+                            target = locators.nth(j)
+                            if await target.is_visible():
+                                logger.info(f"Closing popup: {selector}")
+                                await target.click(timeout=1000)
+                                await page.wait_for_timeout(500)
+                except Exception:
+                    pass
+            
+            # Hammer Escape key
             try:
-                if await page.locator(popup_selector).count() > 0:
+                await page.keyboard.press("Escape")
-                    logger.info(f"Found popup close button: {popup_selector}")
+                await page.wait_for_timeout(200)
-                    await page.locator(popup_selector).first.click(timeout=2000)
+            except: pass
-                    await page.wait_for_timeout(1000)
-            except Exception:
-                pass
 
-        try:
+        # 2. Javascript cleanup (Hide pesky overlays and CMPs that won't close)
-            await page.keyboard.press("Escape")
+        logger.info("Injecting CSS/JS cleanup for persistent overlays...")
-        except Exception:
+        await page.evaluate("""
-            pass
+            () => {
+                const selectorsToHide = [
+                    '#didomi-host', '.didomi-popup-container', '[id*="didomi"]',
+                    '#onetrust-banner-sdk', '#onetrust-consent-sdk',
+                    '.cookie-banner', '.cookie-consent', '.qc-cmp2-container',
+                    '.modal-backdrop', '.modal-overlay', '.fade.show'
+                ];
+                
+                // Hide specific IDs/classes
+                selectorsToHide.forEach(s => {
+                    const el = document.querySelector(s);
+                    if (el) el.style.display = 'none';
+                });
+
+                // Clear overlays (fixed/absolute elements with high z-index that cover too much)
+                const all = document.getElementsByTagName("*");
+                for (let i = 0, max = all.length; i < max; i++) {
+                    const el = all[i];
+                    const style = window.getComputedStyle(el);
+                    if (style.position === 'fixed' || style.position === 'absolute') {
+                        const zIndex = parseInt(style.zIndex);
+                        if (zIndex > 100) {
+                            // Check if it's potentially a popup (covers a lot of area or has modal classes)
+                            const rect = el.getBoundingClientRect();
+                            if (rect.width > window.innerWidth * 0.5 && rect.height > window.innerHeight * 0.5) {
+                                console.log('Hiding potential popup:', el);
+                                el.style.setProperty('display', 'none', 'important');
+                            }
+                        }
+                    }
+                }
+                
+                // Unlock scroll if it was locked by a modal
+                document.body.style.overflow = 'auto';
+                document.documentElement.style.overflow = 'auto';
+            }
+        """)
+        
+        # Wait for any transitions
+        await page.wait_for_timeout(1000)
 
     @staticmethod
     async def _wait_for_selector(page: Page, selector: str):