From 5516562cb68c181430c77e650c1b7a44023514b2 Mon Sep 17 00:00:00 2001
From: Michael SCHAL <michaelschal@gmail.com>
Date: Tue, 2 Dec 2025 09:27:12 +0100
Subject: [PATCH] fix: Handle Amazon interstitials and dynamic screenshot
 updates

- Added Amazon 'Continue' interstitial selectors to BrowserlessService
- Ported robust popup handling from TrackingScraperService to BrowserlessService
- Updated ItemService to fetch dynamic screenshot URLs from PriceHistory
  (fixes issue where forced updates didn't show new screenshots)
---
 app/services/browserless_service.py | 40 ++++++++++++++++++++++++-----
 app/services/item_service.py        | 31 +++++++++++++++-------
 2 files changed, 56 insertions(+), 15 deletions(-)

diff --git a/app/services/browserless_service.py b/app/services/browserless_service.py
index 99596f3..73922f2 100644
--- a/app/services/browserless_service.py
+++ b/app/services/browserless_service.py
@@ -49,6 +49,13 @@ POPUP_SELECTORS = [
     "button:has-text('Tout accepter')",
     "button:has-text('Accepter')",
     "input[aria-labelledby='sp-cc-accept-label']",
+    "#nav-flyout-prime button",
+    
+    # Amazon Interstitials (Soft blocks)
+    "button:has-text('Continuer les achats')",
+    "input[value='Continuer les achats']",
+    "span.a-button-inner > input.a-button-input[type='submit']",
+    "[aria-labelledby='continue-shopping-label']",
 ]
 
 
@@ -193,21 +200,42 @@ class BrowserlessService:
 
     @staticmethod
     async def _handle_popups(page: Page):
-        """Attempt to close popups and cookie banners."""
+        """Attempt to close popups and cookie banners with retry logic."""
         logger.debug("Attempting to close popups...")
+        
+        # First pass: Try to close all visible popups
+        closed_popups = []
         for popup_selector in POPUP_SELECTORS:
             try:
                 if await page.locator(popup_selector).count() > 0:
                     logger.info(f"🚫 Closing popup: {popup_selector}")
                     await page.locator(popup_selector).first.click(timeout=2000)
-                    await page.wait_for_timeout(500)
+                    closed_popups.append(popup_selector)
+                    await page.wait_for_timeout(1000)
             except Exception:
                 pass
 
-        try:
-            await page.keyboard.press("Escape")
-        except Exception:
-            pass
+        # Try Escape key multiple times
+        for _ in range(2):
+            try:
+                await page.keyboard.press("Escape")
+                await page.wait_for_timeout(500)
+            except Exception:
+                pass
+                
+        # Second pass: Verify and retry if any popups are still visible
+        if closed_popups:
+            logger.info(f"Verifying {len(closed_popups)} closed popup(s)...")
+            await page.wait_for_timeout(1000)
+            
+            for popup_selector in closed_popups:
+                try:
+                    if await page.locator(popup_selector).count() > 0:
+                        logger.warning(f"Popup reappeared, retrying: {popup_selector}")
+                        await page.locator(popup_selector).first.click(timeout=2000)
+                        await page.wait_for_timeout(1000)
+                except Exception:
+                    pass
 
     @staticmethod
     async def _extract_amazon_price(page: Page) -> str:
diff --git a/app/services/item_service.py b/app/services/item_service.py
index 4fa7c4f..d809cd8 100644
--- a/app/services/item_service.py
+++ b/app/services/item_service.py
@@ -16,15 +16,28 @@ class ItemService:
     @staticmethod
     def get_items(db: Session):
         items = db.query(models.Item).all()
-        return [
-            {
-                **item.__dict__,
-                "screenshot_url": f"/screenshots/item_{item.id}.png"
-                if os.path.exists(f"screenshots/item_{item.id}.png")
-                else None,
-            }
-            for item in items
-        ]
+        result = []
+        for item in items:
+            # Get latest price history with screenshot
+            latest_history = (
+                db.query(models.PriceHistory)
+                .filter(models.PriceHistory.item_id == item.id)
+                .filter(models.PriceHistory.screenshot_path.isnot(None))
+                .order_by(models.PriceHistory.timestamp.desc())
+                .first()
+            )
+            
+            screenshot_url = None
+            if latest_history and latest_history.screenshot_path:
+                # Convert absolute path /app/screenshots/... to URL /screenshots/...
+                filename = os.path.basename(latest_history.screenshot_path)
+                screenshot_url = f"/screenshots/{filename}"
+            elif os.path.exists(f"screenshots/item_{item.id}.png"):
+                 # Fallback to legacy static file
+                 screenshot_url = f"/screenshots/item_{item.id}.png"
+
+            result.append({**item.__dict__, "screenshot_url": screenshot_url})
+        return result
 
     @staticmethod
     def create_item(db: Session, item: schemas.ItemCreate):