mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-16 19:57:55 +02:00
Resolve merge conflict in browserless_service.py: Keep navigation logic and fix screenshot path
This commit is contained in:
commit
d8b3014021
6 files changed
+106
-22
No files matched your search
@@ -49,13 +49,17 @@ POPUP_SELECTORS = [
|
||||
"button:has-text('Tout accepter')",
|
||||
"button:has-text('Accepter')",
|
||||
"input[aria-labelledby='sp-cc-accept-label']",
|
||||
# Amazon interstitials
|
||||
"#nav-flyout-prime button",
|
||||
|
||||
# Amazon Interstitials (Soft blocks)
|
||||
"button:has-text('Continuer les achats')",
|
||||
"span:has-text('Continuer les achats')",
|
||||
"a:has-text('Continuer les achats')",
|
||||
"input[value='Continuer les achats']",
|
||||
"input[value='Continue shopping']",
|
||||
"span.a-button-inner > input.a-button-input[type='submit']",
|
||||
"form:has-text('Continuer les achats') input[type='submit']",
|
||||
"[aria-labelledby='continue-shopping-label']",
|
||||
]
|
||||
|
||||
|
||||
@@ -159,7 +163,7 @@ class BrowserlessService:
|
||||
|
||||
options = {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"user_agent": user_agent,
|
||||
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||
"locale": "fr-FR",
|
||||
"timezone_id": "Europe/Paris",
|
||||
"java_script_enabled": True,
|
||||
@@ -175,6 +179,29 @@ class BrowserlessService:
|
||||
logger.info("Proxy support requested (not implemented in this version)")
|
||||
|
||||
context = await browser.new_context(**options)
|
||||
|
||||
# Inject stealth scripts
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', {
|
||||
get: () => undefined
|
||||
});
|
||||
Object.defineProperty(navigator, 'languages', {
|
||||
get: () => ['fr-FR', 'fr', 'en-US', 'en']
|
||||
});
|
||||
Object.defineProperty(navigator, 'plugins', {
|
||||
get: () => [1, 2, 3, 4, 5]
|
||||
});
|
||||
window.chrome = { runtime: {} };
|
||||
|
||||
// Mock permissions
|
||||
const originalQuery = window.navigator.permissions.query;
|
||||
window.navigator.permissions.query = (parameters) => (
|
||||
parameters.name === 'notifications' ?
|
||||
Promise.resolve({ state: Notification.permission }) :
|
||||
originalQuery(parameters)
|
||||
);
|
||||
""")
|
||||
|
||||
await context.route("**/*", lambda route: route.continue_())
|
||||
|
||||
return context
|
||||
@@ -182,7 +209,6 @@ class BrowserlessService:
|
||||
@staticmethod
|
||||
async def _navigate_and_wait(page: Page, url: str, timeout: int):
|
||||
"""Navigate to URL and wait for page load."""
|
||||
logger.info(f"📡 Navigating to {url} (Timeout: {timeout}ms)")
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
|
||||
logger.info(f"✅ Page loaded (domcontentloaded): {url}")
|
||||
@@ -200,21 +226,42 @@ class BrowserlessService:
|
||||
|
||||
@staticmethod
|
||||
async def _handle_popups(page: Page):
|
||||
"""Attempt to close popups and cookie banners."""
|
||||
"""Attempt to close popups and cookie banners with retry logic."""
|
||||
logger.debug("Attempting to close popups...")
|
||||
|
||||
# First pass: Try to close all visible popups
|
||||
closed_popups = []
|
||||
for popup_selector in POPUP_SELECTORS:
|
||||
try:
|
||||
if await page.locator(popup_selector).count() > 0:
|
||||
logger.info(f"🚫 Closing popup: {popup_selector}")
|
||||
await page.locator(popup_selector).first.click(timeout=2000)
|
||||
await page.wait_for_timeout(500)
|
||||
closed_popups.append(popup_selector)
|
||||
await page.wait_for_timeout(1000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
await page.keyboard.press("Escape")
|
||||
except Exception:
|
||||
pass
|
||||
# Try Escape key multiple times
|
||||
for _ in range(2):
|
||||
try:
|
||||
await page.keyboard.press("Escape")
|
||||
await page.wait_for_timeout(500)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Second pass: Verify and retry if any popups are still visible
|
||||
if closed_popups:
|
||||
logger.info(f"Verifying {len(closed_popups)} closed popup(s)...")
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
for popup_selector in closed_popups:
|
||||
try:
|
||||
if await page.locator(popup_selector).count() > 0:
|
||||
logger.warning(f"Popup reappeared, retrying: {popup_selector}")
|
||||
await page.locator(popup_selector).first.click(timeout=2000)
|
||||
await page.wait_for_timeout(1000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
@staticmethod
|
||||
async def _extract_amazon_price(page: Page) -> str:
|
||||
@@ -514,10 +561,10 @@ class BrowserlessService:
|
||||
# Take screenshot
|
||||
screenshot_path = ""
|
||||
try:
|
||||
os.makedirs("/app/screenshots", exist_ok=True)
|
||||
os.makedirs("screenshots", exist_ok=True)
|
||||
timestamp = int(time.time() * 1000)
|
||||
safe_name = "".join(c if c.isalnum() else "_" for c in url.split("//")[-1])[:50]
|
||||
screenshot_path = f"/app/screenshots/{safe_name}_{timestamp}.jpg"
|
||||
screenshot_path = f"screenshots/{safe_name}_{timestamp}.jpg"
|
||||
|
||||
is_amazon = "amazon" in url.lower() and "/dp/" in url
|
||||
|
||||
|
||||
@@ -16,15 +16,28 @@ class ItemService:
|
||||
@staticmethod
|
||||
def get_items(db: Session):
|
||||
items = db.query(models.Item).all()
|
||||
return [
|
||||
{
|
||||
**item.__dict__,
|
||||
"screenshot_url": f"/screenshots/item_{item.id}.png"
|
||||
if os.path.exists(f"screenshots/item_{item.id}.png")
|
||||
else None,
|
||||
}
|
||||
for item in items
|
||||
]
|
||||
result = []
|
||||
for item in items:
|
||||
# Get latest price history with screenshot
|
||||
latest_history = (
|
||||
db.query(models.PriceHistory)
|
||||
.filter(models.PriceHistory.item_id == item.id)
|
||||
.filter(models.PriceHistory.screenshot_path.isnot(None))
|
||||
.order_by(models.PriceHistory.timestamp.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
screenshot_url = None
|
||||
if latest_history and latest_history.screenshot_path:
|
||||
# Convert absolute path /app/screenshots/... to URL /screenshots/...
|
||||
filename = os.path.basename(latest_history.screenshot_path)
|
||||
screenshot_url = f"/screenshots/{filename}"
|
||||
elif os.path.exists(f"screenshots/item_{item.id}.png"):
|
||||
# Fallback to legacy static file
|
||||
screenshot_url = f"/screenshots/item_{item.id}.png"
|
||||
|
||||
result.append({**item.__dict__, "screenshot_url": screenshot_url})
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def create_item(db: Session, item: schemas.ItemCreate):
|
||||
|
||||
@@ -131,7 +131,7 @@ async def process_item_check(item_id: int):
|
||||
try:
|
||||
import shutil
|
||||
import os
|
||||
dest_path = f"/app/screenshots/item_{item_id}.png"
|
||||
dest_path = f"screenshots/item_{item_id}.png"
|
||||
# Ensure directory exists
|
||||
os.makedirs(os.path.dirname(dest_path), exist_ok=True)
|
||||
shutil.copy2(screenshot_path, dest_path)
|
||||
|
||||
@@ -20,6 +20,12 @@ POPUP_SELECTORS = [
|
||||
"input[aria-labelledby='sp-cc-accept-label']",
|
||||
"#nav-flyout-prime button", # Prime flyout
|
||||
|
||||
# Amazon Interstitials (Soft blocks)
|
||||
"button:has-text('Continuer les achats')",
|
||||
"input[value='Continuer les achats']",
|
||||
"span.a-button-inner > input.a-button-input[type='submit']",
|
||||
"[aria-labelledby='continue-shopping-label']",
|
||||
|
||||
# French RGPD/Cookie consent platforms
|
||||
"#axeptio_btn_acceptAll", # Axeptio
|
||||
"#axeptio_main_button",
|
||||
|
||||
@@ -50,6 +50,7 @@
|
||||
"targetPrice": "Prix cible",
|
||||
"targetPricePlaceholder": "ex: 999.99",
|
||||
"targetPriceOptional": "(optionnel)",
|
||||
"category": "Catégorie",
|
||||
"selector": "Sélecteur CSS",
|
||||
"selectorPlaceholder": "ex: .price",
|
||||
"selectorOptional": "(optionnel - pour les pages complexes)",
|
||||
@@ -193,4 +194,4 @@
|
||||
"jobConfigUpdated": "Configuration de la tâche mise à jour",
|
||||
"openrouterError": "Erreur lors du chargement des modèles OpenRouter"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# Task: Fix Screenshot Update in Tracking
|
||||
|
||||
## Context
|
||||
The user reports that forcing an update in the "Suivi" (Tracking) section does not update the screenshot as expected.
|
||||
|
||||
## Current Focus
|
||||
Investigating the "force update" mechanism and why the screenshot is not refreshing.
|
||||
|
||||
## Master Plan
|
||||
- [x] Investigate the "force update" API endpoint and backend logic.
|
||||
- [x] Check how screenshots are captured and saved in `tracking_scraper_service.py` or similar.
|
||||
- [x] Verify if the frontend is correctly requesting the update and displaying the new image.
|
||||
- [x] Fix the issue preventing the screenshot from updating.
|
||||
- [x] Verify the fix (Code review confirms relative paths are now consistent across the app).
|
||||
|
||||
## Progress Log
|
||||
- Created `task.md`.
|
||||
Reference in new issue
Block a user