From d50b18690e3c7f057c79f8f5b5b52cea9bc09d5e Mon Sep 17 00:00:00 2001 From: R0m1k3 Date: Thu, 18 Dec 2025 12:38:32 +0100 Subject: [PATCH] feat: Add scheduler and scraper services for automated item tracking and price extraction. --- app/services/scheduler_service.py | 8 +++- app/services/tracking_scraper_service.py | 56 ++++++++++++++++++++---- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index 29a4d25..ce3cf12 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -107,13 +107,19 @@ async def process_item_check(item_id: int): # Use independent ScraperService for tracking from app.services.tracking_scraper_service import ScraperService - screenshot_path, html_content = await ScraperService.scrape_item( + screenshot_path, html_content, final_url = await ScraperService.scrape_item( url=item_data["url"], selector=item_data["selector"], item_id=item_id, return_html=True ) + # Detect Amazon login wall/redirection + if "amazon" in item_data["url"] and ("signin" in final_url or "captcha" in final_url): + logger.warning(f"Amazon Bot Detection triggered for item {item_id}. URL redirected to: {final_url}") + await loop.run_in_executor(None, _update_db_error, item_id, "Amazon Bot Detection: Redirected to login/captcha") + return + if not screenshot_path: raise Exception("Failed to capture screenshot") diff --git a/app/services/tracking_scraper_service.py b/app/services/tracking_scraper_service.py index 1ce0f21..d9a5897 100644 --- a/app/services/tracking_scraper_service.py +++ b/app/services/tracking_scraper_service.py @@ -130,10 +130,10 @@ class ScraperService: item_id: int | None = None, config: ScrapeConfig | None = None, return_html: bool = False, - ) -> tuple[str | None, str]: + ) -> tuple[str | None, str, str]: """ Scrapes the given URL using Browserless and Playwright. - Returns a tuple: (screenshot_path, page_text_or_html) + Returns a tuple: (screenshot_path, page_text_or_html, final_url) """ if config is None: config = ScrapeConfig() @@ -145,14 +145,16 @@ class ScraperService: # Ensure browser is connected and healthy if not await ScraperService._ensure_browser_connected(): logger.error("Failed to establish browser connection") - return None, "" + return None, "", url try: - context = await ScraperService._create_context(ScraperService._browser) + context = await ScraperService._create_context(ScraperService._browser, url) page = await context.new_page() try: await ScraperService._navigate_and_wait(page, url, timeout) + final_url = page.url + await ScraperService._handle_popups(page) if selector: @@ -170,14 +172,14 @@ class ScraperService: screenshot_path = await ScraperService._take_screenshot(page, url, item_id) - return screenshot_path, content_data + return screenshot_path, content_data, final_url finally: await context.close() except Exception as e: logger.error(f"Error scraping {url}: {e}") - return None, "" + return None, "", url @staticmethod async def _connect_browser(p) -> Browser: @@ -185,8 +187,13 @@ class ScraperService: return await p.chromium.connect_over_cdp(BROWSERLESS_URL) @staticmethod - async def _create_context(browser: Browser) -> BrowserContext: - """Create context with stealth and locale (aligned with ImprovedSearchService)""" + async def _create_context(browser: Browser, url: str) -> BrowserContext: + """Create context with advanced stealth and headers (specifically for Amazon)""" + # Determine base domain for referer + from urllib.parse import urlparse + parsed = urlparse(url) + base_domain = f"{parsed.scheme}://{parsed.netloc}/" + context = await browser.new_context( viewport={"width": 1920, "height": 1080}, user_agent=( @@ -196,12 +203,43 @@ class ScraperService: ), locale="fr-FR", timezone_id="Europe/Paris", + extra_http_headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7", + "Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7", + "Cache-Control": "max-age=0", + "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"', + "Sec-Ch-Ua-Mobile": "?0", + "Sec-Ch-Ua-Platform": '"Windows"', + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", + "Referer": base_domain if "amazon" in url else "https://www.google.com/" + } ) - # Stealth mode + # Advanced Stealth mode await context.add_init_script(""" + // Redefine webdriver Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); + + // Masquerade plugins + Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] }); + + // Languages + Object.defineProperty(navigator, 'languages', { get: () => ['fr-FR', 'fr', 'en-US', 'en'] }); + + // Chrome runtime window.chrome = { runtime: {} }; + + // Overwrite permissions + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: Notification.permission }) : + originalQuery(parameters) + ); """) await context.route("**/*", lambda route: route.continue_())