mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-11 17:29:14 +02:00
feat: Add Amazon France search page with Crawl4AI anti-detection
- Created Amazon scraper service with advanced anti-bot techniques: * User-Agent rotation from realistic pool * Complete browser headers (Accept, Accept-Language, etc.) * Proxy rotation (10 residential proxies) * Random delays (1.5-4s) to mimic human behavior * Crawl4AI browser fingerprint randomization * NetworkIdle waiting for complete page load * Cookie acceptance automation - Added Amazon search API endpoint with SSE streaming * Real-time progress updates * Proper error handling * Health check endpoint - Created dedicated Amazon France frontend page: * Modern UI with product cards * Rating display (stars + review count) * Price formatting with discount badges * Prime badge support * Stock status indicators * Sponsored product labels * Direct Amazon links - Removed store list (ENSEIGNES_DATA cleared) * Migration from discount stores to Amazon France * Catalog system kept for future use - Updated navigation: * Added "Amazon France" menu item with ShoppingBag icon * Positioned between Search and Compare * Available on desktop and mobile Technical stack: - Backend: Crawl4AI + BeautifulSoup for scraping - Frontend: React + Shadcn UI components - API: FastAPI with SSE streaming
This commit is contained in:
7 files changed
+887
-77
No files matched your search
+4
-1
@@ -13,7 +13,7 @@ from sqlalchemy import text
|
||||
|
||||
from app.database import SessionLocal, engine
|
||||
from app.limiter import limiter
|
||||
from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug, catalogues
|
||||
from app.routers import auth, items, jobs, notifications, openrouter, search, search_sites, settings, debug, catalogues, amazon
|
||||
from app.services.scheduler_service import scheduled_refresh, scheduler
|
||||
from app.services import auth_service, search_service, seed_enseignes
|
||||
from app.services.scheduler import start_scheduler as start_catalog_scheduler, stop_scheduler as stop_catalog_scheduler
|
||||
@@ -190,6 +190,9 @@ app.include_router(catalogues.router)
|
||||
# Auth router (already has /api prefix)
|
||||
app.include_router(auth.router)
|
||||
|
||||
# Amazon router (already has /api prefix)
|
||||
app.include_router(amazon.router)
|
||||
|
||||
|
||||
@app.get("/api/")
|
||||
def read_root():
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
"""
|
||||
API Router pour la recherche Amazon France
|
||||
Utilise Server-Sent Events (SSE) pour le streaming des résultats
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import AsyncGenerator
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from app.services.amazon_scraper import scrape_amazon_search, AmazonProduct
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/api/amazon", tags=["amazon"])
|
||||
|
||||
|
||||
class AmazonSearchProgress(BaseModel):
|
||||
"""Progress model for SSE streaming"""
|
||||
status: str # 'searching', 'completed', 'error'
|
||||
total: int
|
||||
completed: int
|
||||
message: str
|
||||
results: list[dict]
|
||||
|
||||
|
||||
@router.get("/search")
|
||||
async def search_amazon(
|
||||
q: str = Query(..., min_length=1, description="Terme de recherche"),
|
||||
max_results: int = Query(20, ge=1, le=50, description="Nombre max de résultats"),
|
||||
):
|
||||
"""
|
||||
Recherche de produits sur Amazon France.
|
||||
|
||||
Retourne un flux SSE avec les résultats.
|
||||
|
||||
Format des événements SSE:
|
||||
- event: progress
|
||||
- data: {"status": "...", "total": 1, "completed": 0/1, "results": [...]}
|
||||
"""
|
||||
|
||||
async def generate() -> AsyncGenerator[str, None]:
|
||||
try:
|
||||
# Initial progress
|
||||
progress = AmazonSearchProgress(
|
||||
status="searching",
|
||||
total=1,
|
||||
completed=0,
|
||||
message=f"Recherche sur Amazon France: '{q}'...",
|
||||
results=[]
|
||||
)
|
||||
yield f"event: progress\ndata: {progress.model_dump_json()}\n\n"
|
||||
|
||||
# Execute search
|
||||
logger.info(f"Starting Amazon search for: {q}")
|
||||
products = await scrape_amazon_search(q, max_results=max_results)
|
||||
|
||||
# Convert to dict
|
||||
results = [p.model_dump() for p in products]
|
||||
|
||||
# Final progress
|
||||
if results:
|
||||
progress = AmazonSearchProgress(
|
||||
status="completed",
|
||||
total=1,
|
||||
completed=1,
|
||||
message=f"✅ {len(results)} produits trouvés",
|
||||
results=results
|
||||
)
|
||||
else:
|
||||
progress = AmazonSearchProgress(
|
||||
status="completed",
|
||||
total=1,
|
||||
completed=1,
|
||||
message="Aucun produit trouvé",
|
||||
results=[]
|
||||
)
|
||||
|
||||
yield f"event: progress\ndata: {progress.model_dump_json()}\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error during Amazon search: {e}", exc_info=True)
|
||||
error_progress = AmazonSearchProgress(
|
||||
status="error",
|
||||
total=1,
|
||||
completed=1,
|
||||
message=f"Erreur: {str(e)}",
|
||||
results=[]
|
||||
)
|
||||
yield f"event: progress\ndata: {error_progress.model_dump_json()}\n\n"
|
||||
|
||||
return StreamingResponse(
|
||||
generate(),
|
||||
media_type="text/event-stream",
|
||||
headers={
|
||||
"Cache-Control": "no-cache",
|
||||
"Connection": "keep-alive",
|
||||
"X-Accel-Buffering": "no",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def amazon_health():
|
||||
"""Vérifie que le service Amazon est opérationnel"""
|
||||
return {
|
||||
"status": "ok",
|
||||
"service": "Amazon France Scraper",
|
||||
"anti_detection": "enabled",
|
||||
"features": [
|
||||
"User-Agent rotation",
|
||||
"Proxy rotation (10 proxies)",
|
||||
"Random delays",
|
||||
"Realistic headers",
|
||||
"Crawl4AI anti-detection"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
"""
|
||||
Amazon France Scraper Service - Powered by Crawl4AI
|
||||
Anti-bot detection techniques:
|
||||
- Realistic User-Agent rotation
|
||||
- Complete HTTP headers mimicking real browsers
|
||||
- Proxy rotation (10 residential proxies)
|
||||
- Random delays between requests
|
||||
- Browser fingerprint randomization
|
||||
- Cookie persistence
|
||||
- NetworkIdle waiting for complete page load
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from app.core.search_config import AMAZON_PROXY_LIST_RAW, USER_AGENT_POOL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Amazon France configuration
|
||||
AMAZON_FR_BASE_URL = "https://www.amazon.fr"
|
||||
AMAZON_FR_SEARCH_URL = "https://www.amazon.fr/s?k={query}"
|
||||
|
||||
# Additional realistic User-Agents specifically for Amazon
|
||||
AMAZON_USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0",
|
||||
]
|
||||
|
||||
# Realistic browser headers to avoid bot detection
|
||||
def get_realistic_headers(user_agent: str) -> dict:
|
||||
"""Generate realistic browser headers"""
|
||||
return {
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",
|
||||
"Accept-Language": "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Cache-Control": "max-age=0",
|
||||
"User-Agent": user_agent,
|
||||
}
|
||||
|
||||
|
||||
def get_random_proxy() -> dict | None:
|
||||
"""Get a random proxy from the pool"""
|
||||
if not AMAZON_PROXY_LIST_RAW:
|
||||
return None
|
||||
|
||||
proxy_str = random.choice(AMAZON_PROXY_LIST_RAW)
|
||||
parts = proxy_str.split(":")
|
||||
|
||||
if len(parts) == 4:
|
||||
return {
|
||||
"server": f"http://{parts[0]}:{parts[1]}",
|
||||
"username": parts[2],
|
||||
"password": parts[3]
|
||||
}
|
||||
return None
|
||||
|
||||
|
||||
async def random_delay(min_seconds: float = 1.5, max_seconds: float = 4.0):
|
||||
"""Add random delay to mimic human behavior"""
|
||||
delay = random.uniform(min_seconds, max_seconds)
|
||||
logger.debug(f"Human-like delay: {delay:.2f}s")
|
||||
await asyncio.sleep(delay)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# PYDANTIC SCHEMAS
|
||||
# ============================================================================
|
||||
|
||||
class AmazonProduct(BaseModel):
|
||||
"""Schema for Amazon product extraction"""
|
||||
title: str = Field(description="Product title")
|
||||
url: str = Field(description="Product URL")
|
||||
price: float | None = Field(default=None, description="Price in EUR")
|
||||
original_price: float | None = Field(default=None, description="Original price if discounted")
|
||||
rating: float | None = Field(default=None, description="Product rating (0-5)")
|
||||
reviews_count: int | None = Field(default=None, description="Number of reviews")
|
||||
image_url: str | None = Field(default=None, description="Product image URL")
|
||||
in_stock: bool = Field(default=True, description="Availability status")
|
||||
prime: bool = Field(default=False, description="Prime eligible")
|
||||
sponsored: bool = Field(default=False, description="Is sponsored")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# PRICE PARSING HELPERS
|
||||
# ============================================================================
|
||||
|
||||
def parse_amazon_price(price_text: str) -> float | None:
|
||||
"""
|
||||
Parse Amazon price formats:
|
||||
- "12,99 €"
|
||||
- "12,99€"
|
||||
- "12.99 EUR"
|
||||
- "1 234,99 €"
|
||||
"""
|
||||
if not price_text:
|
||||
return None
|
||||
|
||||
# Remove currency symbols and extra spaces
|
||||
cleaned = price_text.strip().replace('€', '').replace('EUR', '').strip()
|
||||
|
||||
# Remove thousands separators (space or dot in French format)
|
||||
cleaned = cleaned.replace(' ', '').replace('\xa0', '') # \xa0 is non-breaking space
|
||||
|
||||
# Replace comma with dot for decimal separator
|
||||
cleaned = cleaned.replace(',', '.')
|
||||
|
||||
# Extract first number (in case of ranges like "12.99 - 15.99")
|
||||
match = re.search(r'(\d+\.?\d*)', cleaned)
|
||||
if match:
|
||||
try:
|
||||
return float(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_rating(rating_text: str) -> float | None:
|
||||
"""Parse rating like '4,5 sur 5 étoiles' or '4.5 out of 5 stars'"""
|
||||
if not rating_text:
|
||||
return None
|
||||
|
||||
# Match patterns like "4,5" or "4.5"
|
||||
match = re.search(r'(\d+[,.]\d+)', rating_text)
|
||||
if match:
|
||||
try:
|
||||
return float(match.group(1).replace(',', '.'))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def parse_reviews_count(reviews_text: str) -> int | None:
|
||||
"""Parse review count like '1 234' or '12,345'"""
|
||||
if not reviews_text:
|
||||
return None
|
||||
|
||||
# Remove non-digit characters except spaces
|
||||
cleaned = re.sub(r'[^\d\s]', '', reviews_text)
|
||||
cleaned = cleaned.replace(' ', '').replace('\xa0', '')
|
||||
|
||||
try:
|
||||
return int(cleaned)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# SCRAPING FUNCTIONS
|
||||
# ============================================================================
|
||||
|
||||
async def scrape_amazon_search(query: str, max_results: int = 20) -> list[AmazonProduct]:
|
||||
"""
|
||||
Scrape Amazon France search results for a given query.
|
||||
|
||||
Anti-detection features:
|
||||
1. Random User-Agent from realistic pool
|
||||
2. Complete browser headers
|
||||
3. Random proxy from pool
|
||||
4. Random delays between operations
|
||||
5. Browser fingerprint randomization via Crawl4AI
|
||||
6. NetworkIdle waiting
|
||||
7. Cookie handling
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
max_results: Maximum number of products to return (default 20)
|
||||
|
||||
Returns:
|
||||
List of AmazonProduct objects
|
||||
"""
|
||||
search_url = AMAZON_FR_SEARCH_URL.format(query=quote_plus(query))
|
||||
logger.info(f"🔍 Searching Amazon France: {query}")
|
||||
logger.info(f"📍 URL: {search_url}")
|
||||
|
||||
# Select random User-Agent
|
||||
user_agent = random.choice(AMAZON_USER_AGENTS)
|
||||
logger.debug(f"🎭 User-Agent: {user_agent[:50]}...")
|
||||
|
||||
# Select random proxy
|
||||
proxy = get_random_proxy()
|
||||
if proxy:
|
||||
logger.debug(f"🌐 Using proxy: {proxy['server']}")
|
||||
else:
|
||||
logger.warning("⚠️ No proxy available - may face rate limiting")
|
||||
|
||||
# Configure Crawl4AI browser with anti-detection
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
verbose=False,
|
||||
user_agent=user_agent,
|
||||
proxy=proxy,
|
||||
extra_args=[
|
||||
"--disable-blink-features=AutomationControlled", # Disable automation detection
|
||||
"--disable-dev-shm-usage",
|
||||
"--no-sandbox",
|
||||
"--disable-gpu",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-web-security",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
"--disable-infobars",
|
||||
"--window-size=1920,1080",
|
||||
"--start-maximized",
|
||||
# Randomize viewport
|
||||
f"--user-agent={user_agent}",
|
||||
],
|
||||
)
|
||||
|
||||
# Configure crawler behavior
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS, # Always fetch fresh data
|
||||
wait_for_images=True,
|
||||
process_iframes=False,
|
||||
remove_overlay_elements=True, # Remove popups/modals
|
||||
wait_until="networkidle", # Wait for all network requests
|
||||
delay_before_return_html=2.0, # Extra wait for JS rendering
|
||||
page_timeout=30000, # 30 seconds timeout
|
||||
# Accept cookies automatically
|
||||
js_code="""
|
||||
// Accept cookies if banner appears
|
||||
const cookieButton = document.querySelector('#sp-cc-accept, button[id*="accept"]');
|
||||
if (cookieButton) {
|
||||
cookieButton.click();
|
||||
}
|
||||
""",
|
||||
)
|
||||
|
||||
products = []
|
||||
|
||||
try:
|
||||
# Add human-like delay before request
|
||||
await random_delay(1.0, 2.5)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
logger.info("🚀 Launching browser...")
|
||||
result = await crawler.arun(url=search_url, config=crawler_config)
|
||||
|
||||
if not result.success:
|
||||
logger.error(f"❌ Crawl failed: {result.error_message}")
|
||||
return []
|
||||
|
||||
logger.info(f"✅ Page loaded successfully ({len(result.html)} bytes)")
|
||||
|
||||
# Parse HTML with BeautifulSoup
|
||||
soup = BeautifulSoup(result.html, 'html.parser')
|
||||
|
||||
# Amazon uses data-component-type="s-search-result" for product cards
|
||||
product_cards = soup.find_all('div', {'data-component-type': 's-search-result'})
|
||||
|
||||
if not product_cards:
|
||||
logger.warning("⚠️ No products found - checking for CAPTCHA or blocks")
|
||||
# Check for CAPTCHA
|
||||
if 'captcha' in result.html.lower():
|
||||
logger.error("🚫 CAPTCHA detected - Amazon blocked the request")
|
||||
elif 'robot' in result.html.lower() or 'bot' in result.html.lower():
|
||||
logger.error("🤖 Bot detection triggered")
|
||||
else:
|
||||
logger.warning("📦 Empty results - query may have no matches")
|
||||
return []
|
||||
|
||||
logger.info(f"📦 Found {len(product_cards)} product cards")
|
||||
|
||||
for idx, card in enumerate(product_cards):
|
||||
if len(products) >= max_results:
|
||||
break
|
||||
|
||||
try:
|
||||
# Extract ASIN (Amazon Standard Identification Number)
|
||||
asin = card.get('data-asin', '')
|
||||
if not asin:
|
||||
continue
|
||||
|
||||
# Check if sponsored
|
||||
sponsored = bool(card.select_one('[data-component-type="sp-sponsored-result"]'))
|
||||
|
||||
# Extract title
|
||||
title_elem = card.select_one('h2 a span, h2 span')
|
||||
if not title_elem:
|
||||
continue
|
||||
title = title_elem.get_text(strip=True)
|
||||
|
||||
# Extract URL
|
||||
link_elem = card.select_one('h2 a')
|
||||
if not link_elem:
|
||||
continue
|
||||
href = link_elem.get('href', '')
|
||||
product_url = f"{AMAZON_FR_BASE_URL}{href}" if href.startswith('/') else href
|
||||
|
||||
# Extract price
|
||||
price = None
|
||||
original_price = None
|
||||
|
||||
# Current price
|
||||
price_elem = card.select_one('.a-price .a-offscreen')
|
||||
if price_elem:
|
||||
price = parse_amazon_price(price_elem.get_text(strip=True))
|
||||
|
||||
# Original price (if discounted)
|
||||
original_price_elem = card.select_one('.a-price.a-text-price .a-offscreen')
|
||||
if original_price_elem:
|
||||
original_price = parse_amazon_price(original_price_elem.get_text(strip=True))
|
||||
|
||||
# Extract rating
|
||||
rating = None
|
||||
rating_elem = card.select_one('[aria-label*="étoile"], [aria-label*="star"]')
|
||||
if rating_elem:
|
||||
rating = parse_rating(rating_elem.get('aria-label', ''))
|
||||
|
||||
# Extract reviews count
|
||||
reviews_count = None
|
||||
reviews_elem = card.select_one('[aria-label*="étoile"] + span, [aria-label*="star"] + span')
|
||||
if reviews_elem:
|
||||
reviews_count = parse_reviews_count(reviews_elem.get_text(strip=True))
|
||||
|
||||
# Extract image
|
||||
image_url = None
|
||||
img_elem = card.select_one('img.s-image')
|
||||
if img_elem:
|
||||
image_url = img_elem.get('src') or img_elem.get('data-src')
|
||||
|
||||
# Check Prime eligibility
|
||||
prime = bool(card.select_one('[aria-label*="Prime"], i.a-icon-prime'))
|
||||
|
||||
# Check availability
|
||||
in_stock = True
|
||||
unavailable_elem = card.select_one('[aria-label*="Indisponible"], [aria-label*="Unavailable"]')
|
||||
if unavailable_elem:
|
||||
in_stock = False
|
||||
|
||||
product = AmazonProduct(
|
||||
title=title,
|
||||
url=product_url,
|
||||
price=price,
|
||||
original_price=original_price,
|
||||
rating=rating,
|
||||
reviews_count=reviews_count,
|
||||
image_url=image_url,
|
||||
in_stock=in_stock,
|
||||
prime=prime,
|
||||
sponsored=sponsored,
|
||||
)
|
||||
|
||||
products.append(product)
|
||||
logger.debug(f" ✓ [{idx+1}] {title[:50]}... - {price}€")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error parsing product card {idx}: {e}")
|
||||
continue
|
||||
|
||||
logger.info(f"✅ Successfully extracted {len(products)} products")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"❌ Error during Amazon scraping: {e}", exc_info=True)
|
||||
return []
|
||||
|
||||
return products
|
||||
|
||||
|
||||
async def scrape_amazon_search_batched(
|
||||
query: str,
|
||||
max_results: int = 20,
|
||||
batch_size: int = 20
|
||||
) -> list[AmazonProduct]:
|
||||
"""
|
||||
Scrape Amazon with automatic pagination if needed.
|
||||
|
||||
Note: For now, we just scrape the first page (20 results).
|
||||
Pagination can be added later if needed.
|
||||
|
||||
Args:
|
||||
query: Search query
|
||||
max_results: Maximum total results (default 20)
|
||||
batch_size: Results per page (default 20)
|
||||
|
||||
Returns:
|
||||
List of AmazonProduct objects
|
||||
"""
|
||||
# For now, single page only
|
||||
return await scrape_amazon_search(query, max_results)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# TESTING / VERIFICATION
|
||||
# ============================================================================
|
||||
|
||||
async def test_amazon_scraper():
|
||||
"""Test the Amazon scraper with a simple query"""
|
||||
logger.info("=" * 60)
|
||||
logger.info("Testing Amazon France Scraper")
|
||||
logger.info("=" * 60)
|
||||
|
||||
test_query = "aspirateur"
|
||||
products = await scrape_amazon_search(test_query, max_results=5)
|
||||
|
||||
logger.info(f"\n📊 Results for '{test_query}':")
|
||||
logger.info(f"Found {len(products)} products\n")
|
||||
|
||||
for idx, product in enumerate(products, 1):
|
||||
logger.info(f"{idx}. {product.title}")
|
||||
logger.info(f" 💰 Price: {product.price}€" + (f" (was {product.original_price}€)" if product.original_price else ""))
|
||||
logger.info(f" ⭐ Rating: {product.rating}/5 ({product.reviews_count} reviews)" if product.rating else " ⭐ No rating")
|
||||
logger.info(f" 🔗 {product.url}")
|
||||
logger.info(f" {'✅ Prime' if product.prime else '📦 Standard'} | {'📢 Sponsored' if product.sponsored else '🔍 Organic'}")
|
||||
logger.info("")
|
||||
|
||||
return products
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run test
|
||||
asyncio.run(test_amazon_scraper())
|
||||
@@ -9,81 +9,9 @@ from app.models import Enseigne
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Data for the 9 enseignes
|
||||
ENSEIGNES_DATA = [
|
||||
{
|
||||
"nom": "Gifi",
|
||||
"slug_bonial": "Gifi",
|
||||
"couleur": "#E30613",
|
||||
"site_url": "https://www.gifi.fr",
|
||||
"description": "Décoration, maison, bazar",
|
||||
"ordre_affichage": 1,
|
||||
},
|
||||
{
|
||||
"nom": "Action",
|
||||
"slug_bonial": "Action",
|
||||
"couleur": "#0066B3",
|
||||
"site_url": "https://www.action.com/fr-fr/",
|
||||
"description": "Discount non-alimentaire",
|
||||
"ordre_affichage": 2,
|
||||
},
|
||||
{
|
||||
"nom": "Centrakor",
|
||||
"slug_bonial": "Centrakor",
|
||||
"couleur": "#E94E1B",
|
||||
"site_url": "https://www.centrakor.com",
|
||||
"description": "Décoration, maison",
|
||||
"ordre_affichage": 3,
|
||||
},
|
||||
{
|
||||
"nom": "La Foir'Fouille",
|
||||
"slug_bonial": "La-Foir-Fouille",
|
||||
"couleur": "#009639",
|
||||
"site_url": "https://www.lafoirfouille.fr",
|
||||
"description": "Bazar, décoration",
|
||||
"ordre_affichage": 4,
|
||||
},
|
||||
{
|
||||
"nom": "Stokomani",
|
||||
"slug_bonial": "Stokomani",
|
||||
"couleur": "#FF6600",
|
||||
"site_url": "https://www.stokomani.fr",
|
||||
"description": "Déstockage textile et maison",
|
||||
"ordre_affichage": 5,
|
||||
},
|
||||
{
|
||||
"nom": "B&M",
|
||||
"slug_bonial": "BM",
|
||||
"couleur": "#D4145A",
|
||||
"site_url": "https://bmstores.fr",
|
||||
"description": "Discount britannique",
|
||||
"ordre_affichage": 6,
|
||||
},
|
||||
{
|
||||
"nom": "L'Incroyable",
|
||||
"slug_bonial": "L-incroyable",
|
||||
"couleur": "#8B0000",
|
||||
"site_url": "https://www.lincroyable.fr",
|
||||
"description": "Décoration et mobilier discount (Groupe Althys, siège à Denain)",
|
||||
"ordre_affichage": 7,
|
||||
},
|
||||
{
|
||||
"nom": "Bazarland",
|
||||
"slug_bonial": "Bazarland",
|
||||
"couleur": "#FFCC00",
|
||||
"site_url": None,
|
||||
"description": "Bazar discount",
|
||||
"ordre_affichage": 8,
|
||||
},
|
||||
{
|
||||
"nom": "Noz",
|
||||
"slug_bonial": "Noz",
|
||||
"couleur": "#003366",
|
||||
"site_url": "https://www.noz.fr",
|
||||
"description": "Déstockage généraliste",
|
||||
"ordre_affichage": 9,
|
||||
},
|
||||
]
|
||||
# Liste des enseignes vidée - migration vers Amazon France uniquement
|
||||
# Les magasins discount ne sont plus gérés via le système de catalogues
|
||||
ENSEIGNES_DATA = []
|
||||
|
||||
|
||||
def seed_enseignes(db: Session) -> int:
|
||||
|
||||
@@ -5,6 +5,7 @@ import Layout from '@/components/layout/Layout'
|
||||
import Dashboard from '@/pages/Dashboard'
|
||||
import Search from '@/pages/Search'
|
||||
import MultiSearch from '@/pages/MultiSearch'
|
||||
import AmazonSearch from '@/pages/AmazonSearch'
|
||||
import Catalogues from '@/pages/Catalogues'
|
||||
import Login from '@/pages/Login'
|
||||
import Admin from '@/pages/Admin'
|
||||
@@ -78,6 +79,7 @@ function AppRoutes() {
|
||||
<Route path="/" element={<Dashboard />} />
|
||||
<Route path="/search" element={<Search />} />
|
||||
<Route path="/compare" element={<MultiSearch />} />
|
||||
<Route path="/amazon" element={<AmazonSearch />} />
|
||||
<Route path="/catalogues" element={<Catalogues />} />
|
||||
<Route
|
||||
path="/admin"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import React, { useState } from 'react';
|
||||
import { Link, useLocation, useNavigate } from 'react-router-dom';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { LayoutDashboard, Search, TrendingUp, BookOpen, Moon, Sun, Menu, LogOut, Shield } from 'lucide-react';
|
||||
import { LayoutDashboard, Search, TrendingUp, BookOpen, Moon, Sun, Menu, LogOut, Shield, ShoppingBag } from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Sheet, SheetContent, SheetTrigger } from '@/components/ui/sheet';
|
||||
import { cn } from '@/lib/utils';
|
||||
@@ -22,6 +22,7 @@ const Layout = ({ children, theme, toggleTheme }) => {
|
||||
const navItems = [
|
||||
{ icon: LayoutDashboard, label: t('nav.dashboard'), path: '/' },
|
||||
{ icon: Search, label: t('nav.search') || 'Recherche', path: '/search' },
|
||||
{ icon: ShoppingBag, label: 'Amazon France', path: '/amazon' },
|
||||
{ icon: TrendingUp, label: 'Comparateur', path: '/compare' },
|
||||
{ icon: BookOpen, label: 'Catalogues', path: '/catalogues' },
|
||||
];
|
||||
|
||||
@@ -0,0 +1,323 @@
|
||||
import React, { useState, useRef } from 'react';
|
||||
import { toast } from 'sonner';
|
||||
import { useTranslation } from 'react-i18next';
|
||||
import { Search as SearchIcon, Loader2, Star, ShoppingCart, ExternalLink, Tag } from 'lucide-react';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Card, CardContent } from '@/components/ui/card';
|
||||
import { Progress } from '@/components/ui/progress';
|
||||
import { Badge } from '@/components/ui/badge';
|
||||
|
||||
const API_URL = '/api';
|
||||
|
||||
export default function AmazonSearch() {
|
||||
const { t } = useTranslation();
|
||||
const [query, setQuery] = useState('');
|
||||
const [isSearching, setIsSearching] = useState(false);
|
||||
const [progress, setProgress] = useState(null);
|
||||
const [results, setResults] = useState([]);
|
||||
const eventSourceRef = useRef(null);
|
||||
|
||||
const handleSearch = async (e) => {
|
||||
e.preventDefault();
|
||||
if (!query.trim()) {
|
||||
toast.error('Veuillez entrer un terme de recherche');
|
||||
return;
|
||||
}
|
||||
|
||||
// Fermer l'EventSource précédent si existant
|
||||
if (eventSourceRef.current) {
|
||||
eventSourceRef.current.close();
|
||||
}
|
||||
|
||||
setIsSearching(true);
|
||||
setResults([]);
|
||||
setProgress({ status: 'searching', total: 1, completed: 0 });
|
||||
|
||||
// Construire l'URL avec les paramètres
|
||||
const params = new URLSearchParams({
|
||||
q: query.trim(),
|
||||
max_results: 20,
|
||||
});
|
||||
|
||||
// Créer l'EventSource pour SSE
|
||||
const eventSource = new EventSource(`${API_URL}/amazon/search?${params}`);
|
||||
eventSourceRef.current = eventSource;
|
||||
|
||||
eventSource.addEventListener('progress', (event) => {
|
||||
try {
|
||||
const data = JSON.parse(event.data);
|
||||
setProgress(data);
|
||||
setResults(data.results || []);
|
||||
|
||||
if (data.status === 'completed' || data.status === 'error') {
|
||||
setIsSearching(false);
|
||||
eventSource.close();
|
||||
|
||||
if (data.status === 'completed') {
|
||||
toast.success(data.message);
|
||||
} else if (data.message) {
|
||||
toast.error(data.message);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Error parsing SSE data:', error);
|
||||
}
|
||||
});
|
||||
|
||||
eventSource.onerror = () => {
|
||||
setIsSearching(false);
|
||||
eventSource.close();
|
||||
toast.error('Erreur de connexion au serveur');
|
||||
};
|
||||
};
|
||||
|
||||
const progressPercent = progress
|
||||
? Math.round((progress.completed / Math.max(progress.total, 1)) * 100)
|
||||
: 0;
|
||||
|
||||
const formatPrice = (price, originalPrice) => {
|
||||
if (!price) return 'Prix indisponible';
|
||||
|
||||
const formattedPrice = price.toFixed(2);
|
||||
if (originalPrice && originalPrice > price) {
|
||||
const discount = Math.round(((originalPrice - price) / originalPrice) * 100);
|
||||
return (
|
||||
<div className="flex items-baseline gap-2">
|
||||
<span className="text-2xl font-bold text-green-600">{formattedPrice}€</span>
|
||||
<span className="text-sm text-muted-foreground line-through">{originalPrice.toFixed(2)}€</span>
|
||||
<Badge variant="destructive" className="text-xs">-{discount}%</Badge>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return <span className="text-2xl font-bold">{formattedPrice}€</span>;
|
||||
};
|
||||
|
||||
const renderRating = (rating, reviewsCount) => {
|
||||
if (!rating) return null;
|
||||
|
||||
return (
|
||||
<div className="flex items-center gap-1">
|
||||
<div className="flex">
|
||||
{[...Array(5)].map((_, i) => (
|
||||
<Star
|
||||
key={i}
|
||||
className={`h-4 w-4 ${
|
||||
i < Math.floor(rating)
|
||||
? 'fill-yellow-400 text-yellow-400'
|
||||
: 'text-gray-300'
|
||||
}`}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
<span className="text-sm font-medium">{rating.toFixed(1)}</span>
|
||||
{reviewsCount && (
|
||||
<span className="text-sm text-muted-foreground">
|
||||
({reviewsCount.toLocaleString('fr-FR')})
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div className="space-y-6 animate-in fade-in duration-500">
|
||||
{/* Header */}
|
||||
<div className="flex flex-col gap-4 md:flex-row md:items-center md:justify-between">
|
||||
<div>
|
||||
<div className="flex items-center gap-3">
|
||||
<h2 className="text-2xl font-bold tracking-tight">Amazon France</h2>
|
||||
<Badge variant="outline" className="bg-orange-50 border-orange-200">
|
||||
🇫🇷 France
|
||||
</Badge>
|
||||
</div>
|
||||
<p className="text-muted-foreground mt-1">
|
||||
Recherchez parmi des millions de produits sur Amazon.fr
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Search Form */}
|
||||
<Card>
|
||||
<CardContent className="pt-6">
|
||||
<form onSubmit={handleSearch} className="space-y-4">
|
||||
<div className="flex gap-2">
|
||||
<div className="relative flex-1">
|
||||
<SearchIcon className="absolute left-3 top-3 h-4 w-4 text-muted-foreground" />
|
||||
<Input
|
||||
type="text"
|
||||
placeholder="Rechercher un produit sur Amazon..."
|
||||
value={query}
|
||||
onChange={(e) => setQuery(e.target.value)}
|
||||
className="pl-9"
|
||||
disabled={isSearching}
|
||||
/>
|
||||
</div>
|
||||
<Button type="submit" disabled={isSearching || !query.trim()}>
|
||||
{isSearching ? (
|
||||
<>
|
||||
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
|
||||
Recherche...
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<SearchIcon className="h-4 w-4 mr-2" />
|
||||
Rechercher
|
||||
</>
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* Info */}
|
||||
<div className="flex items-center gap-2 text-sm text-muted-foreground">
|
||||
<div className="flex items-center gap-1">
|
||||
<div className="h-2 w-2 rounded-full bg-green-500"></div>
|
||||
<span>Anti-détection activé</span>
|
||||
</div>
|
||||
<span>•</span>
|
||||
<span>Max 20 résultats</span>
|
||||
</div>
|
||||
</form>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
{/* Progress Bar */}
|
||||
{progress && isSearching && (
|
||||
<Card>
|
||||
<CardContent className="pt-6">
|
||||
<div className="space-y-2">
|
||||
<div className="flex justify-between text-sm">
|
||||
<span>{progress.message || 'Recherche en cours...'}</span>
|
||||
<span>{progressPercent}%</span>
|
||||
</div>
|
||||
<Progress value={progressPercent} />
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* Results */}
|
||||
{results.length > 0 && (
|
||||
<div className="space-y-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<h3 className="text-lg font-semibold">
|
||||
{results.length} produit{results.length > 1 ? 's' : ''} trouvé{results.length > 1 ? 's' : ''}
|
||||
</h3>
|
||||
</div>
|
||||
<div className="grid gap-4 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4">
|
||||
{results.map((product, index) => (
|
||||
<Card key={index} className="overflow-hidden hover:shadow-lg transition-shadow">
|
||||
<CardContent className="p-0">
|
||||
{/* Image */}
|
||||
{product.image_url && (
|
||||
<div className="relative aspect-square bg-gray-50 flex items-center justify-center p-4">
|
||||
<img
|
||||
src={product.image_url}
|
||||
alt={product.title}
|
||||
className="max-h-full max-w-full object-contain"
|
||||
/>
|
||||
{product.sponsored && (
|
||||
<Badge
|
||||
variant="secondary"
|
||||
className="absolute top-2 left-2 text-xs"
|
||||
>
|
||||
Sponsorisé
|
||||
</Badge>
|
||||
)}
|
||||
{!product.in_stock && (
|
||||
<Badge
|
||||
variant="destructive"
|
||||
className="absolute top-2 right-2 text-xs"
|
||||
>
|
||||
Indisponible
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="p-4 space-y-3">
|
||||
{/* Title */}
|
||||
<h4 className="font-medium text-sm line-clamp-2 min-h-[2.5rem]">
|
||||
{product.title}
|
||||
</h4>
|
||||
|
||||
{/* Rating */}
|
||||
{renderRating(product.rating, product.reviews_count)}
|
||||
|
||||
{/* Price */}
|
||||
<div className="pt-2">
|
||||
{formatPrice(product.price, product.original_price)}
|
||||
</div>
|
||||
|
||||
{/* Badges */}
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{product.prime && (
|
||||
<Badge variant="default" className="bg-blue-500 text-xs">
|
||||
Prime
|
||||
</Badge>
|
||||
)}
|
||||
{product.in_stock && (
|
||||
<Badge variant="outline" className="text-xs text-green-600 border-green-600">
|
||||
En stock
|
||||
</Badge>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Actions */}
|
||||
<div className="pt-2 flex gap-2">
|
||||
<Button
|
||||
variant="default"
|
||||
size="sm"
|
||||
className="flex-1"
|
||||
onClick={() => window.open(product.url, '_blank')}
|
||||
>
|
||||
<ShoppingCart className="h-4 w-4 mr-1" />
|
||||
Voir
|
||||
</Button>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => window.open(product.url, '_blank')}
|
||||
>
|
||||
<ExternalLink className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Empty State */}
|
||||
{!isSearching && results.length === 0 && !progress && (
|
||||
<Card>
|
||||
<CardContent className="py-12 text-center">
|
||||
<div className="mx-auto mb-4 h-16 w-16 rounded-full bg-orange-50 flex items-center justify-center">
|
||||
<SearchIcon className="h-8 w-8 text-orange-500" />
|
||||
</div>
|
||||
<h3 className="text-lg font-medium mb-2">Recherchez sur Amazon France</h3>
|
||||
<p className="text-muted-foreground max-w-md mx-auto">
|
||||
Entrez un terme de recherche pour trouver des produits sur Amazon.fr.
|
||||
Notre système anti-détection garantit un accès fiable aux résultats.
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{/* No Results */}
|
||||
{!isSearching && results.length === 0 && progress?.status === 'completed' && (
|
||||
<Card>
|
||||
<CardContent className="py-12 text-center">
|
||||
<SearchIcon className="h-12 w-12 mx-auto text-muted-foreground mb-4" />
|
||||
<h3 className="text-lg font-medium mb-2">Aucun produit trouvé</h3>
|
||||
<p className="text-muted-foreground">
|
||||
Essayez avec d'autres termes de recherche
|
||||
</p>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in new issue
Block a user