Resolve merge conflict in browserless_service.py: Keep navigation logic and fix screenshot path

This commit is contained in:
Michael committed 2025-12-02 17:04:49 +01:00
commit d8b3014021
6 files changed
+106 -22

No files matched your search

+58 -11
View File
@@ -49,13 +49,17 @@ POPUP_SELECTORS = [
"button:has-text('Tout accepter')",
"button:has-text('Accepter')",
"input[aria-labelledby='sp-cc-accept-label']",
# Amazon interstitials
"#nav-flyout-prime button",
# Amazon Interstitials (Soft blocks)
"button:has-text('Continuer les achats')",
"span:has-text('Continuer les achats')",
"a:has-text('Continuer les achats')",
"input[value='Continuer les achats']",
"input[value='Continue shopping']",
"span.a-button-inner > input.a-button-input[type='submit']",
"form:has-text('Continuer les achats') input[type='submit']",
"[aria-labelledby='continue-shopping-label']",
]
@@ -159,7 +163,7 @@ class BrowserlessService:
options = {
"viewport": {"width": 1920, "height": 1080},
"user_agent": user_agent,
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
"locale": "fr-FR",
"timezone_id": "Europe/Paris",
"java_script_enabled": True,
@@ -175,6 +179,29 @@ class BrowserlessService:
logger.info("Proxy support requested (not implemented in this version)")
context = await browser.new_context(**options)
# Inject stealth scripts
await context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', {
get: () => undefined
});
Object.defineProperty(navigator, 'languages', {
get: () => ['fr-FR', 'fr', 'en-US', 'en']
});
Object.defineProperty(navigator, 'plugins', {
get: () => [1, 2, 3, 4, 5]
});
window.chrome = { runtime: {} };
// Mock permissions
const originalQuery = window.navigator.permissions.query;
window.navigator.permissions.query = (parameters) => (
parameters.name === 'notifications' ?
Promise.resolve({ state: Notification.permission }) :
originalQuery(parameters)
);
""")
await context.route("**/*", lambda route: route.continue_())
return context
@@ -182,7 +209,6 @@ class BrowserlessService:
@staticmethod
async def _navigate_and_wait(page: Page, url: str, timeout: int):
"""Navigate to URL and wait for page load."""
logger.info(f"📡 Navigating to {url} (Timeout: {timeout}ms)")
try:
await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
logger.info(f"✅ Page loaded (domcontentloaded): {url}")
@@ -200,21 +226,42 @@ class BrowserlessService:
@staticmethod
async def _handle_popups(page: Page):
"""Attempt to close popups and cookie banners."""
"""Attempt to close popups and cookie banners with retry logic."""
logger.debug("Attempting to close popups...")
# First pass: Try to close all visible popups
closed_popups = []
for popup_selector in POPUP_SELECTORS:
try:
if await page.locator(popup_selector).count() > 0:
logger.info(f"🚫 Closing popup: {popup_selector}")
await page.locator(popup_selector).first.click(timeout=2000)
await page.wait_for_timeout(500)
closed_popups.append(popup_selector)
await page.wait_for_timeout(1000)
except Exception:
pass
try:
await page.keyboard.press("Escape")
except Exception:
pass
# Try Escape key multiple times
for _ in range(2):
try:
await page.keyboard.press("Escape")
await page.wait_for_timeout(500)
except Exception:
pass
# Second pass: Verify and retry if any popups are still visible
if closed_popups:
logger.info(f"Verifying {len(closed_popups)} closed popup(s)...")
await page.wait_for_timeout(1000)
for popup_selector in closed_popups:
try:
if await page.locator(popup_selector).count() > 0:
logger.warning(f"Popup reappeared, retrying: {popup_selector}")
await page.locator(popup_selector).first.click(timeout=2000)
await page.wait_for_timeout(1000)
except Exception:
pass
@staticmethod
async def _extract_amazon_price(page: Page) -> str:
@@ -514,10 +561,10 @@ class BrowserlessService:
# Take screenshot
screenshot_path = ""
try:
os.makedirs("/app/screenshots", exist_ok=True)
os.makedirs("screenshots", exist_ok=True)
timestamp = int(time.time() * 1000)
safe_name = "".join(c if c.isalnum() else "_" for c in url.split("//")[-1])[:50]
screenshot_path = f"/app/screenshots/{safe_name}_{timestamp}.jpg"
screenshot_path = f"screenshots/{safe_name}_{timestamp}.jpg"
is_amazon = "amazon" in url.lower() and "/dp/" in url
+22 -9
View File
@@ -16,15 +16,28 @@ class ItemService:
@staticmethod
def get_items(db: Session):
items = db.query(models.Item).all()
return [
{
**item.__dict__,
"screenshot_url": f"/screenshots/item_{item.id}.png"
if os.path.exists(f"screenshots/item_{item.id}.png")
else None,
}
for item in items
]
result = []
for item in items:
# Get latest price history with screenshot
latest_history = (
db.query(models.PriceHistory)
.filter(models.PriceHistory.item_id == item.id)
.filter(models.PriceHistory.screenshot_path.isnot(None))
.order_by(models.PriceHistory.timestamp.desc())
.first()
)
screenshot_url = None
if latest_history and latest_history.screenshot_path:
# Convert absolute path /app/screenshots/... to URL /screenshots/...
filename = os.path.basename(latest_history.screenshot_path)
screenshot_url = f"/screenshots/{filename}"
elif os.path.exists(f"screenshots/item_{item.id}.png"):
# Fallback to legacy static file
screenshot_url = f"/screenshots/item_{item.id}.png"
result.append({**item.__dict__, "screenshot_url": screenshot_url})
return result
@staticmethod
def create_item(db: Session, item: schemas.ItemCreate):
+1 -1
View File
@@ -131,7 +131,7 @@ async def process_item_check(item_id: int):
try:
import shutil
import os
dest_path = f"/app/screenshots/item_{item_id}.png"
dest_path = f"screenshots/item_{item_id}.png"
# Ensure directory exists
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
shutil.copy2(screenshot_path, dest_path)
+6
View File
@@ -20,6 +20,12 @@ POPUP_SELECTORS = [
"input[aria-labelledby='sp-cc-accept-label']",
"#nav-flyout-prime button", # Prime flyout
# Amazon Interstitials (Soft blocks)
"button:has-text('Continuer les achats')",
"input[value='Continuer les achats']",
"span.a-button-inner > input.a-button-input[type='submit']",
"[aria-labelledby='continue-shopping-label']",
# French RGPD/Cookie consent platforms
"#axeptio_btn_acceptAll", # Axeptio
"#axeptio_main_button",
+2 -1
View File
@@ -50,6 +50,7 @@
"targetPrice": "Prix cible",
"targetPricePlaceholder": "ex: 999.99",
"targetPriceOptional": "(optionnel)",
"category": "Catégorie",
"selector": "Sélecteur CSS",
"selectorPlaceholder": "ex: .price",
"selectorOptional": "(optionnel - pour les pages complexes)",
@@ -193,4 +194,4 @@
"jobConfigUpdated": "Configuration de la tâche mise à jour",
"openrouterError": "Erreur lors du chargement des modèles OpenRouter"
}
}
}
+17
View File
@@ -0,0 +1,17 @@
# Task: Fix Screenshot Update in Tracking
## Context
The user reports that forcing an update in the "Suivi" (Tracking) section does not update the screenshot as expected.
## Current Focus
Investigating the "force update" mechanism and why the screenshot is not refreshing.
## Master Plan
- [x] Investigate the "force update" API endpoint and backend logic.
- [x] Check how screenshots are captured and saved in `tracking_scraper_service.py` or similar.
- [x] Verify if the frontend is correctly requesting the update and displaying the new image.
- [x] Fix the issue preventing the screenshot from updating.
- [x] Verify the fix (Code review confirms relative paths are now consistent across the app).
## Progress Log
- Created `task.md`.