Michael/Priceflow
1
0 Fork 0
mirror of https://github.com/R0m1k3/Priceflow.git synced 2026-10-23 07:08:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #171 from R0m1k3/antigravity

Browse Source 
Antigravity
This commit is contained in:
LogiFlow authored and GitHub committed 2025-11-30 13:25:41 +01:00
2 parents ca0eaf67b9 b8a56ca85d
commit fb9b98787d
1 file changed
+150 -87
Download Patch File Download Diff File Expand all files Collapse all files
app/services/browserless_service.py
+150 -87
View File
@@ -1,10 +1,13 @@
""" """
Browserless Service - Enhanced Version Browserless Service - Enhanced Persistent Browser Version
Manages connections to Browserless.io with advanced robustness features: Inspired by AmazonScraperService architecture for better performance and reliability
- Auto-reconnection on browser disconnection
- Advanced popup handling Features:
- Flexible configuration - Persistent browser connection (singleton pattern)
- Thread-safe operations - Auto-reconnection on disconnection
- Thread-safe operations (asyncio.Lock)
- Improved price extraction accuracy
- Optimized timeouts and screenshots
- Amazon & Generic price extraction - Amazon & Generic price extraction
""" """
@@ -45,6 +48,7 @@ POPUP_SELECTORS = [
"#onetrust-accept-btn-handler", "#onetrust-accept-btn-handler",
"button:has-text('Tout accepter')", "button:has-text('Tout accepter')",
"button:has-text('Accepter')", "button:has-text('Accepter')",
"input[aria-labelledby='sp-cc-accept-label']",
] ]
@@ -55,11 +59,14 @@ class ScrapeConfig:
smart_scroll: bool = False smart_scroll: bool = False
scroll_pixels: int = 350 scroll_pixels: int = 350
text_length: int = 0 text_length: int = 0
timeout: int = 90000 timeout: int = 30000 # Reduced from 90s to 30s
class BrowserlessService: class BrowserlessService:
"""Enhanced browserless service with auto-reconnection and robust error handling.""" """
Enhanced persistent browserless service with auto-reconnection.
Architecture inspired by AmazonScraperService for optimal performance.
"""
_playwright = None _playwright = None
_browser: Browser | None = None _browser: Browser | None = None
@@ -75,10 +82,10 @@ class BrowserlessService:
async def _initialize(cls): async def _initialize(cls):
"""Internal initialization logic (Assumes lock is held).""" """Internal initialization logic (Assumes lock is held)."""
if cls._browser is None: if cls._browser is None:
logger.info("Initializing BrowserlessService shared browser...") logger.info("🚀 Initializing BrowserlessService persistent browser...")
cls._playwright = await async_playwright().start() cls._playwright = await async_playwright().start()
cls._browser = await cls._connect_browser(cls._playwright) cls._browser = await cls._connect_browser(cls._playwright)
logger.info("BrowserlessService initialized.") logger.info("✅ BrowserlessService browser ready (persistent)")
@classmethod @classmethod
async def shutdown(cls): async def shutdown(cls):
@@ -95,7 +102,10 @@ class BrowserlessService:
@classmethod @classmethod
async def _ensure_browser_connected(cls) -> bool: async def _ensure_browser_connected(cls) -> bool:
"""Ensure browser is connected, reconnect if needed.""" """
Ensure browser is connected, reconnect if needed.
Returns True if browser is ready, False otherwise.
"""
async with cls._lock: async with cls._lock:
try: try:
if cls._browser is None: if cls._browser is None:
@@ -107,10 +117,13 @@ class BrowserlessService:
try: try:
test_context = await cls._browser.new_context() test_context = await cls._browser.new_context()
await test_context.close() await test_context.close()
logger.debug("✅ Browser connection test passed (reusing)")
return True return True
except Exception as e: except Exception as e:
logger.error(f"Browser connection test failed: {e}") logger.error(f"❌ Browser connection test failed: {e}")
logger.info("Attempting to reconnect browser...") logger.info("🔄 Attempting to reconnect browser...")
# Clear the old browser
cls._browser = None cls._browser = None
if cls._playwright: if cls._playwright:
try: try:
@@ -118,6 +131,8 @@ class BrowserlessService:
except Exception: except Exception:
pass pass
cls._playwright = None cls._playwright = None
# Reconnect
await cls._initialize() await cls._initialize()
return cls._browser is not None return cls._browser is not None
except Exception as e: except Exception as e:
@@ -160,32 +175,32 @@ class BrowserlessService:
@staticmethod @staticmethod
async def _navigate_and_wait(page: Page, url: str, timeout: int): async def _navigate_and_wait(page: Page, url: str, timeout: int):
"""Navigate to URL and wait for page load.""" """Navigate to URL and wait for page load."""
logger.info(f"Navigating to {url} (Timeout: {timeout}ms)") logger.info(f"📡 Navigating to {url} (Timeout: {timeout}ms)")
try: try:
await page.goto(url, wait_until="domcontentloaded", timeout=timeout) await page.goto(url, wait_until="domcontentloaded", timeout=timeout)
logger.info(f"Page loaded (domcontentloaded): {url}") logger.info(f"✅ Page loaded (domcontentloaded): {url}")
try: try:
await page.wait_for_load_state("networkidle", timeout=5000) await page.wait_for_load_state("networkidle", timeout=2000) # Reduced from 5s
logger.info("Network idle reached") logger.info("✅ Network idle reached")
except PlaywrightTimeoutError: except PlaywrightTimeoutError:
logger.info("Network idle timed out (non-critical), proceeding...") logger.debug("⚠️ Network idle timed out (non-critical), proceeding...")
except Exception as e: except Exception as e:
logger.warning(f"Navigation warning for {url}: {e}") logger.warning(f"Navigation warning for {url}: {e}")
await page.wait_for_timeout(2000) await page.wait_for_timeout(1500) # Reduced from 2s
@staticmethod @staticmethod
async def _handle_popups(page: Page): async def _handle_popups(page: Page):
"""Attempt to close popups and cookie banners.""" """Attempt to close popups and cookie banners."""
logger.info("Attempting to close popups...") logger.debug("Attempting to close popups...")
for popup_selector in POPUP_SELECTORS: for popup_selector in POPUP_SELECTORS:
try: try:
if await page.locator(popup_selector).count() > 0: if await page.locator(popup_selector).count() > 0:
logger.info(f"Found popup close button: {popup_selector}") logger.info(f"🚫 Closing popup: {popup_selector}")
await page.locator(popup_selector).first.click(timeout=2000) await page.locator(popup_selector).first.click(timeout=2000)
await page.wait_for_timeout(1000) await page.wait_for_timeout(500)
except Exception: except Exception:
pass pass
@@ -196,9 +211,9 @@ class BrowserlessService:
@staticmethod @staticmethod
async def _extract_amazon_price(page: Page) -> str: async def _extract_amazon_price(page: Page) -> str:
"""Extract price from Amazon product page.""" """Extract price from Amazon product page with strict validation."""
price_selectors = [ price_selectors = [
".a-price .a-offscreen", ".a-price .a-offscreen", # Most reliable
"#corePrice_desktop .a-price .a-offscreen", "#corePrice_desktop .a-price .a-offscreen",
"#corePriceDisplay_desktop_feature_div .a-price .a-offscreen", "#corePriceDisplay_desktop_feature_div .a-price .a-offscreen",
".a-price[data-a-color='price'] .a-offscreen", ".a-price[data-a-color='price'] .a-offscreen",
@@ -212,8 +227,10 @@ class BrowserlessService:
element = page.locator(selector).first element = page.locator(selector).first
price_text = await element.inner_text(timeout=2000) price_text = await element.inner_text(timeout=2000)
if price_text and price_text.strip(): if price_text and price_text.strip():
logger.info(f"Extracted Amazon price via {selector}: {price_text}") # Validate format (Amazon uses "XX,YY €" or "XX,YY")
return price_text.strip() if re.search(r'\d+[.,]\d{2}', price_text):
logger.info(f"💰 Amazon price via {selector}: {price_text}")
return price_text.strip()
except Exception: except Exception:
continue continue
@@ -223,97 +240,138 @@ class BrowserlessService:
fraction = await page.locator("span.a-price-fraction").first.inner_text() fraction = await page.locator("span.a-price-fraction").first.inner_text()
if whole and fraction: if whole and fraction:
price_text = f"{whole}{fraction}" price_text = f"{whole}{fraction}"
logger.info(f"Extracted Amazon price from whole+fraction: {price_text}") logger.info(f"💰 Amazon price from whole+fraction: {price_text}")
return price_text return price_text
except Exception: except Exception:
pass pass
logger.warning("Could not extract Amazon price") logger.warning("⚠️ Could not extract Amazon price")
return "" return ""
@staticmethod @staticmethod
async def _extract_generic_price(page: Page) -> str: async def _extract_generic_price(page: Page) -> str:
"""Extract price from generic e-commerce pages.""" """
price_selectors = [ Extract price from generic e-commerce pages with STRICT validation.
Improvements:
- Strict French price regex: digits,digits €
- Context validation (avoid random numbers)
- Exclude strikethrough prices
"""
# High-priority semantic selectors
high_priority_selectors = [
"[itemprop='price']", "[itemprop='price']",
"[data-testid='price']", "[data-testid='price']",
"[data-test='price']", "[data-test='price']",
"[data-price]",
]
# Medium-priority specific selectors
medium_priority_selectors = [
".current-price", ".current-price",
".sale-price", ".sale-price",
".final-price", ".final-price",
".product-price", ".product-price",
".special-price", ".special-price",
"[data-price]",
".price-current", ".price-current",
".price-now", ".price-now",
".price-sales", ".prix-actuel",
"[class*='price']:not([class*='old']):not([class*='was']):not([class*='original']):not([class*='before']):not([class*='regular']):not([class*='strike']):not([class*='barre'])",
"#price", "#price",
"#product-price", "#product-price",
"span[class*='prix']:not([class*='ancien']):not([class*='barre'])",
".prix-actuel",
".price:not(.old-price):not(.was-price)",
] ]
# Low-priority generic selectors
low_priority_selectors = [
"span[class*='prix']:not([class*='ancien']):not([class*='barre'])",
".price:not(.old-price):not(.was-price)",
"[class*='price']:not([class*='old']):not([class*='was']):not([class*='original']):not([class*='before']):not([class*='strike']):not([class*='barre'])",
]
all_selectors = high_priority_selectors + medium_priority_selectors + low_priority_selectors
found_prices = [] found_prices = []
for selector in price_selectors: # STRICT French price regex: 1-4 digits, comma, 2 digits, optional € symbol
# Examples: "12,99 €", "1 234,99€", "3,50 EUR"
strict_price_regex = re.compile(r'(\d{1,4}(?:\s?\d{3})*[.,]\d{2})\s*€?')
for selector in all_selectors:
try: try:
elements = page.locator(selector) elements = page.locator(selector)
count = await elements.count() count = await elements.count()
for i in range(min(count, 3)): for i in range(min(count, 3)): # Max 3 elements per selector
try: try:
element = elements.nth(i) element = elements.nth(i)
if await element.is_visible(timeout=1000):
# Check if strikethrough # Check visibility
if not await element.is_visible(timeout=1000):
continue
# Check if strikethrough (old price)
try:
text_decoration = await element.evaluate("el => window.getComputedStyle(el).textDecoration")
if "line-through" in text_decoration:
logger.debug(f"⏭️ Skipping strikethrough price at {selector}")
continue
except Exception:
pass
price_text = await element.inner_text()
if not price_text:
continue
# STRICT validation with regex
price_match = strict_price_regex.search(price_text)
if price_match:
matched_price = price_match.group(0)
# Extract numeric value for range validation
numeric_text = matched_price.replace('€', '').replace(' ', '').replace(',', '.')
try: try:
text_decoration = await element.evaluate("el => window.getComputedStyle(el).textDecoration") price_val = float(numeric_text)
if "line-through" in text_decoration:
continue # Reasonable price range: 0.01€ to 100,000€
except Exception: if 0.01 <= price_val <= 100000:
pass found_prices.append((selector, matched_price))
logger.info(f"💰 Valid price via {selector}: {matched_price} ({price_val}€)")
price_text = await element.inner_text() # Return immediately if high-priority selector
if price_text and ('€' in price_text or (',' in price_text and any(c.isdigit() for c in price_text))): if selector in high_priority_selectors:
price_text = price_text.strip() return matched_price
else:
logger.debug(f"⏭️ Price out of range ({price_val}€): {matched_price}")
except ValueError:
logger.debug(f"⏭️ Could not parse price value: {matched_price}")
continue
# Validate price range except Exception as e:
numeric_match = re.search(r'(\d+[.,]?\d*)', price_text.replace(' ', '').replace('\xa0', '')) logger.debug(f"Error processing element {i} of {selector}: {e}")
if numeric_match:
try:
price_val = float(numeric_match.group(1).replace(',', '.'))
if 0.01 <= price_val <= 100000:
found_prices.append((selector, price_text))
logger.info(f"Found valid price via {selector}: {price_text}")
# Return high-priority immediately
if selector in ["[itemprop='price']", "[data-testid='price']", ".current-price", ".sale-price", ".final-price", ".product-price"]:
return price_text
except (ValueError, AttributeError):
continue
except Exception:
continue continue
except Exception: except Exception as e:
logger.debug(f"Error with selector {selector}: {e}")
continue continue
# If multiple prices found, prefer the FIRST valid one (usually current price)
# Changed from LAST to FIRST - more reliable for current price detection
if found_prices: if found_prices:
best_price = found_prices[-1][1] best_price = found_prices[0][1] # Take first (highest priority)
if len(found_prices) > 1: if len(found_prices) > 1:
logger.info(f"Multiple prices found, selecting last: {best_price}") prices_list = [p[1] for p in found_prices]
logger.info(f"💰 Multiple prices found: {prices_list}, selecting first: {best_price}")
return best_price return best_price
# Regex fallback # Fallback: Strict regex search in body text (last resort)
try: try:
all_text = await page.inner_text('body') all_text = await page.inner_text('body')
price_matches = re.findall(r'\d+[,\.]\d{2}\s*€', all_text) price_matches = strict_price_regex.findall(all_text)
if price_matches: if price_matches:
logger.info(f"Found price via regex: {price_matches[0]}") # Take first match
return price_matches[0] first_match = price_matches[0]
except Exception: logger.info(f"💰 Found price via regex fallback: {first_match}")
pass return first_match
except Exception as e:
logger.debug(f"Regex fallback failed: {e}")
logger.warning("Could not extract generic price") logger.warning("⚠️ Could not extract generic price with any method")
return "" return ""
@classmethod @classmethod
@@ -325,7 +383,7 @@ class BrowserlessService:
extract_text: bool = False extract_text: bool = False
) -> tuple[str, str]: ) -> tuple[str, str]:
""" """
Fetch page content with full stealth lifecycle. Fetch page content with persistent browser and optimized performance.
Args: Args:
url: URL to fetch url: URL to fetch
@@ -336,9 +394,9 @@ class BrowserlessService:
Returns: Returns:
tuple[content, screenshot_path]: Content (HTML or text) and screenshot path tuple[content, screenshot_path]: Content (HTML or text) and screenshot path
""" """
# Ensure browser connected # Ensure browser connected (with auto-reconnection)
if not await cls._ensure_browser_connected(): if not await cls._ensure_browser_connected():
logger.error("Failed to establish browser connection") logger.error("❌ Failed to establish browser connection")
return "", "" return "", ""
try: try:
@@ -346,7 +404,8 @@ class BrowserlessService:
page = await context.new_page() page = await context.new_page()
try: try:
await cls._navigate_and_wait(page, url, 90000) # Navigate with optimized timeout (30s instead of 90s)
await cls._navigate_and_wait(page, url, 30000)
await cls._handle_popups(page) await cls._handle_popups(page)
# Amazon-specific wait # Amazon-specific wait
@@ -355,7 +414,7 @@ class BrowserlessService:
for selector in amazon_selectors: for selector in amazon_selectors:
try: try:
await page.wait_for_selector(selector, timeout=5000, state="visible") await page.wait_for_selector(selector, timeout=5000, state="visible")
logger.info(f"Amazon price element found: {selector}") logger.info(f"✅ Amazon price element found: {selector}")
break break
except Exception: except Exception:
continue continue
@@ -363,14 +422,14 @@ class BrowserlessService:
# Extract content # Extract content
if extract_text: if extract_text:
content = await page.inner_text('body') content = await page.inner_text('body')
logger.info(f"Extracted {len(content)} chars of visible text") logger.info(f"📄 Extracted {len(content)} chars of visible text")
# Normalize French prices # Normalize French prices to English format for AI
content = re.sub(r'(\d+)€(\d{2})\b', r'\1.\2 €', content) content = re.sub(r'(\d+)€(\d{2})\b', r'\1.\2 €', content)
content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content) content = re.sub(r'(\d+),(\d{2})\s*€', r'\1.\2 €', content)
content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content) content = re.sub(r'(\d+)\s(\d{3})', r'\1\2', content)
# Extract price # Extract price with improved accuracy
extracted_price = "" extracted_price = ""
if "amazon" in url.lower() and "/dp/" in url: if "amazon" in url.lower() and "/dp/" in url:
extracted_price = await cls._extract_amazon_price(page) extracted_price = await cls._extract_amazon_price(page)
@@ -383,11 +442,14 @@ class BrowserlessService:
normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price) normalized_price = re.sub(r'(\d+),(\d{2})', r'\1.\2', normalized_price)
normalized_price = normalized_price.replace(' ', '') normalized_price = normalized_price.replace(' ', '')
content = f"PRIX DÉTECTÉ: {normalized_price}\n\n{content}" content = f"PRIX DÉTECTÉ: {normalized_price}\n\n{content}"
logger.info(f"Prepended price: {normalized_price}") logger.info(f"💰 Prepended price: {normalized_price}")
else:
logger.warning("⚠️ No price detected for this page")
else: else:
content = await page.content() content = await page.content()
logger.debug(f"📄 Extracted {len(content)} chars of HTML")
# Take screenshot # Take screenshot (optimized - viewport only by default)
screenshot_path = "" screenshot_path = ""
try: try:
os.makedirs("/app/screenshots", exist_ok=True) os.makedirs("/app/screenshots", exist_ok=True)
@@ -404,16 +466,17 @@ class BrowserlessService:
element = page.locator(selector).first element = page.locator(selector).first
if await element.is_visible(): if await element.is_visible():
await element.screenshot(path=screenshot_path, quality=85, type="jpeg") await element.screenshot(path=screenshot_path, quality=85, type="jpeg")
logger.info(f"Amazon focused screenshot: {selector}") logger.info(f"📸 Amazon focused screenshot: {selector}")
break break
except Exception: except Exception:
continue continue
else: else:
await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg") await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg")
else: else:
# Viewport screenshot (faster than full_page)
await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg") await page.screenshot(path=screenshot_path, full_page=False, quality=80, type="jpeg")
logger.info(f"📸 Viewport screenshot saved to {screenshot_path}")
logger.info(f"Screenshot saved to {screenshot_path}")
except Exception as e: except Exception as e:
logger.warning(f"Screenshot failed: {e}") logger.warning(f"Screenshot failed: {e}")
@@ -423,9 +486,9 @@ class BrowserlessService:
await context.close() await context.close()
except Exception as e: except Exception as e:
logger.error(f"Error scraping {url}: {e}") logger.error(f"❌ Error scraping {url}: {e}", exc_info=True)
return "", "" return "", ""
# Global instance # Global instance (backward compatibility)
browserless_service = BrowserlessService() browserless_service = BrowserlessService()