From 7592c2781e3ab01b043a8b09977d1ee16ca93542 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Mon, 1 Dec 2025 09:32:37 +0100 Subject: [PATCH 1/2] feat: Implement the core FastAPI application, including database migrations, service orchestration, and a new tracking scraper service. --- app/main.py | 6 + app/services/scheduler_service.py | 20 +- app/services/tracking_scraper_service.py | 271 +++++++++++++++++++++++ 3 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 app/services/tracking_scraper_service.py diff --git a/app/main.py b/app/main.py index 8690256..323f618 100644 --- a/app/main.py +++ b/app/main.py @@ -19,6 +19,7 @@ from app.services import auth_service, search_service, seed_enseignes from app.services.scheduler import start_scheduler as start_catalog_scheduler, stop_scheduler as stop_catalog_scheduler from app.services.amazon_scraper_service import amazon_scraper_service from app.services.improved_search_service import improved_search_service +from app.services.tracking_scraper_service import TrackingScraperService # Configure logging logging.basicConfig( @@ -150,6 +151,10 @@ async def lifespan(app: FastAPI): logger.info("Initializing Improved Search Service...") await improved_search_service.initialize() + # Initialize Tracking Scraper Service + logger.info("Initializing Tracking Scraper Service...") + await TrackingScraperService.initialize() + logger.info("Application started") yield logger.info("Shutting down schedulers and services...") @@ -157,6 +162,7 @@ async def lifespan(app: FastAPI): stop_catalog_scheduler() await amazon_scraper_service.shutdown() await improved_search_service.shutdown() + await TrackingScraperService.shutdown() logger.info("Application shutdown complete") diff --git a/app/services/scheduler_service.py b/app/services/scheduler_service.py index 67b620a..0844144 100644 --- a/app/services/scheduler_service.py +++ b/app/services/scheduler_service.py @@ -10,6 +10,7 @@ from app.services.ai_service import AIService from app.services.item_service import ItemService from app.services.notification_service import NotificationService from app.services.browserless_service import browserless_service +from app.services.tracking_scraper_service import TrackingScraperService, ScrapeConfig logger = logging.getLogger(__name__) scheduler = AsyncIOScheduler() @@ -103,12 +104,19 @@ async def process_item_check(item_id: int): try: logger.info(f"Checking item: {item_data['name']} ({item_data['url']})") - # Use browserless service with extract_text=True for monitoring - page_text, screenshot_path = await browserless_service.get_page_content( - item_data["url"], - use_proxy="amazon" in item_data["url"], - wait_selector=item_data["selector"], - extract_text=True # Get visible text for AI analysis + # Use TrackingScraperService for monitoring + scrape_config = ScrapeConfig( + smart_scroll=config["smart_scroll"], + scroll_pixels=config["smart_scroll_pixels"], + text_length=config["text_length"], + timeout=config["scraper_timeout"] + ) + + screenshot_path, page_text = await TrackingScraperService.scrape_item( + url=item_data["url"], + selector=item_data["selector"], + item_id=item_id, + config=scrape_config ) # Determine availability based on content presence diff --git a/app/services/tracking_scraper_service.py b/app/services/tracking_scraper_service.py new file mode 100644 index 0000000..9236e6d --- /dev/null +++ b/app/services/tracking_scraper_service.py @@ -0,0 +1,271 @@ +import asyncio +import logging +import os +from dataclasses import dataclass +from datetime import datetime + +from playwright.async_api import Browser, BrowserContext, Page, async_playwright +from playwright.async_api import TimeoutError as PlaywrightTimeoutError + +logger = logging.getLogger(__name__) + +BROWSERLESS_URL = os.getenv("BROWSERLESS_URL", "ws://browserless:3000") + +POPUP_SELECTORS = [ + "button[aria-label='Close']", + "button[aria-label='close']", + ".close-button", + ".modal-close", + "svg[data-name='Close']", + "[class*='popup'] button", + "[class*='modal'] button", + "button:has-text('No, thanks')", + "button:has-text('No thanks')", + "a:has-text('No, thanks')", + "div[role='dialog'] button[aria-label='Close']", +] + + +@dataclass +class ScrapeConfig: + """Configuration for scraping parameters.""" + + smart_scroll: bool = False + scroll_pixels: int = 350 + text_length: int = 0 + timeout: int = 90000 + + +class TrackingScraperService: + _playwright = None + _browser: Browser | None = None + _lock = asyncio.Lock() + + @classmethod + async def initialize(cls): + """Initialize the shared browser instance (Public, Thread-Safe).""" + async with cls._lock: + await cls._initialize() + + @classmethod + async def _initialize(cls): + """Internal initialization logic (Assumes lock is held).""" + if cls._browser is None: + logger.info("Initializing TrackingScraperService shared browser...") + cls._playwright = await async_playwright().start() + cls._browser = await cls._connect_browser(cls._playwright) + logger.info("TrackingScraperService initialized.") + + @classmethod + async def shutdown(cls): + """Shutdown the shared browser instance.""" + async with cls._lock: + if cls._browser: + logger.info("Shutting down TrackingScraperService shared browser...") + await cls._browser.close() + cls._browser = None + if cls._playwright: + await cls._playwright.stop() + cls._playwright = None + logger.info("TrackingScraperService shutdown complete.") + + @classmethod + async def _ensure_browser_connected(cls) -> bool: + """Ensure browser is connected, reconnect if needed.""" + async with cls._lock: + try: + if cls._browser is None: + logger.warning("Browser not initialized, initializing...") + await cls._initialize() + return cls._browser is not None + + # Test if browser is still alive by trying to create a context + try: + test_context = await cls._browser.new_context() + await test_context.close() + return True + except Exception as e: + logger.error(f"Browser connection test failed: {e}") + logger.info("Attempting to reconnect browser...") + # Clear the old browser + cls._browser = None + if cls._playwright: + try: + await cls._playwright.stop() + except Exception: + pass + cls._playwright = None + # Reconnect + await cls._initialize() + return cls._browser is not None + except Exception as e: + logger.error(f"Failed to ensure browser connection: {e}") + return False + + @staticmethod + async def scrape_item( + url: str, + selector: str | None = None, + item_id: int | None = None, + config: ScrapeConfig | None = None, + ) -> tuple[str | None, str]: + """ + Scrapes the given URL using Browserless and Playwright. + Returns a tuple: (screenshot_path, page_text) + """ + if config is None: + config = ScrapeConfig() + + # Input validation & defaults + scroll_pixels = max(350, config.scroll_pixels if config.scroll_pixels > 0 else 350) + timeout = max(30000, config.timeout if config.timeout > 0 else 90000) + + # Ensure browser is connected and healthy + if not await TrackingScraperService._ensure_browser_connected(): + logger.error("Failed to establish browser connection") + return None, "" + + try: + context = await TrackingScraperService._create_context(TrackingScraperService._browser) + page = await context.new_page() + + try: + await TrackingScraperService._navigate_and_wait(page, url, timeout) + await TrackingScraperService._handle_popups(page) + + if selector: + await TrackingScraperService._wait_for_selector(page, selector) + else: + await TrackingScraperService._auto_detect_price(page) + + if config.smart_scroll: + await TrackingScraperService._smart_scroll(page, scroll_pixels) + + page_text = await TrackingScraperService._extract_text(page, config.text_length) + screenshot_path = await TrackingScraperService._take_screenshot(page, url, item_id) + + return screenshot_path, page_text + + finally: + await context.close() + + except Exception as e: + logger.error(f"Error scraping {url}: {e}") + return None, "" + + @staticmethod + async def _connect_browser(p) -> Browser: + logger.info(f"Connecting to Browserless at {BROWSERLESS_URL}") + return await p.chromium.connect_over_cdp(BROWSERLESS_URL) + + @staticmethod + async def _create_context(browser: Browser) -> BrowserContext: + context = await browser.new_context( + viewport={"width": 1920, "height": 1080}, + user_agent=( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/120.0.0.0 Safari/537.36" + ), + ) + # Stealth mode / Ad blocking attempts + await context.route("**/*", lambda route: route.continue_()) + return context + + @staticmethod + async def _navigate_and_wait(page: Page, url: str, timeout: int): + logger.info(f"Navigating to {url} (Timeout: {timeout}ms)") + try: + await page.goto(url, wait_until="domcontentloaded", timeout=timeout) + logger.info(f"Page loaded (domcontentloaded): {url}") + + try: + await page.wait_for_load_state("networkidle", timeout=5000) + logger.info("Network idle reached") + except PlaywrightTimeoutError: + logger.info("Network idle timed out (non-critical), proceeding...") + + except Exception as e: + logger.warning(f"Navigation warning for {url}: {e}") + + # Wait a bit for dynamic content + await page.wait_for_timeout(2000) + + @staticmethod + async def _handle_popups(page: Page): + logger.info("Attempting to close popups...") + for popup_selector in POPUP_SELECTORS: + try: + if await page.locator(popup_selector).count() > 0: + logger.info(f"Found popup close button: {popup_selector}") + await page.locator(popup_selector).first.click(timeout=2000) + await page.wait_for_timeout(1000) + except Exception: + pass + + try: + await page.keyboard.press("Escape") + except Exception: + pass + + @staticmethod + async def _wait_for_selector(page: Page, selector: str): + try: + logger.info(f"Waiting for selector: {selector}") + await page.wait_for_selector(selector, timeout=5000) + element = page.locator(selector).first + await element.scroll_into_view_if_needed() + logger.info(f"Scrolled to selector: {selector}") + except Exception as e: + logger.warning(f"Selector {selector} not found or timed out: {e}") + + @staticmethod + async def _auto_detect_price(page: Page): + logger.info("No selector provided. Attempting to find price element...") + try: + price_locator = page.locator("text=/$[0-9,]+(\\.[0-9]{2})?/") + if await price_locator.count() > 0: + await price_locator.first.scroll_into_view_if_needed() + logger.info("Scrolled to potential price element") + except Exception as e: + logger.warning(f"Auto-price detection failed: {e}") + + @staticmethod + async def _smart_scroll(page: Page, scroll_pixels: int): + logger.info(f"Performing smart scroll ({scroll_pixels}px)...") + try: + await page.evaluate(f"window.scrollBy(0, {scroll_pixels})") + await page.wait_for_timeout(1000) + except Exception as e: + logger.warning(f"Smart scroll failed: {e}") + + @staticmethod + async def _extract_text(page: Page, text_length: int) -> str: + if text_length <= 0: + return "" + + try: + logger.info(f"Extracting text (limit: {text_length} chars)...") + raw_text = await page.inner_text("body") + page_text = raw_text[:text_length] + logger.info(f"Extracted {len(page_text)} characters") + return page_text + except Exception as e: + logger.error(f"Text extraction failed: {e}") + return "" + + @staticmethod + async def _take_screenshot(page: Page, url: str, item_id: int | None) -> str: + screenshot_dir = "screenshots" + os.makedirs(screenshot_dir, exist_ok=True) + + if item_id: + filename = f"{screenshot_dir}/item_{item_id}.png" + else: + url_part = url.split("//")[-1].replace("/", "_") + timestamp = datetime.now().timestamp() + filename = f"{screenshot_dir}/{url_part}_{timestamp}.png" + + await page.screenshot(path=filename, full_page=False) + logger.info(f"Screenshot saved to {filename}") + return filename From f27a75d9d6f3b0733971aadce370ea719ff60742 Mon Sep 17 00:00:00 2001 From: Michael SCHAL Date: Mon, 1 Dec 2025 11:32:12 +0100 Subject: [PATCH 2/2] feat: Implement initial application structure with routing, authentication flow, and responsive layout. --- frontend/src/App.jsx | 7 +- frontend/src/components/layout/Layout.jsx | 219 ++++++++++++---------- frontend/src/index.css | 85 ++++----- 3 files changed, 157 insertions(+), 154 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 134d986..cf7bed8 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -55,12 +55,9 @@ function PublicRoute({ children }) { } function AppRoutes() { - const { theme, toggleTheme } = useTheme(); - const { isAuthenticated } = useAuth(); - return ( <> - + - + } /> } /> diff --git a/frontend/src/components/layout/Layout.jsx b/frontend/src/components/layout/Layout.jsx index 690a190..ed21308 100644 --- a/frontend/src/components/layout/Layout.jsx +++ b/frontend/src/components/layout/Layout.jsx @@ -1,13 +1,13 @@ import React, { useState } from 'react'; import { Link, useLocation, useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; -import { LayoutDashboard, Search, TrendingUp, BookOpen, Moon, Sun, Menu, LogOut, Shield, ShoppingBag } from 'lucide-react'; +import { LayoutDashboard, Search, TrendingUp, BookOpen, Menu, LogOut, Shield, ShoppingBag } from 'lucide-react'; import { Button } from '@/components/ui/button'; import { Sheet, SheetContent, SheetTrigger } from '@/components/ui/sheet'; import { cn } from '@/lib/utils'; import { useAuth } from '@/hooks/use-auth'; -const Layout = ({ children, theme, toggleTheme }) => { +const Layout = ({ children }) => { const { t } = useTranslation(); const location = useLocation(); const navigate = useNavigate(); @@ -19,6 +19,7 @@ const Layout = ({ children, theme, toggleTheme }) => { navigate('/login'); }; + // Main navigation items (without admin) const navItems = [ { icon: LayoutDashboard, label: t('nav.dashboard'), path: '/' }, { icon: Search, label: t('nav.search') || 'Recherche', path: '/search' }, @@ -27,25 +28,23 @@ const Layout = ({ children, theme, toggleTheme }) => { { icon: BookOpen, label: 'Catalogues', path: '/catalogues' }, ]; - // Add admin link for admin users - if (isAdmin) { - navItems.push({ icon: Shield, label: 'Administration', path: '/admin' }); - } - return ( -
- {/* Sidebar */} -