mirror of
https://github.com/R0m1k3/Priceflow.git
synced 2026-10-22 22:58:11 +02:00
feat: add Cataloguemate.fr scraper, replacing Tiendeo and utilizing BrowserlessService for robust scraping.
This commit is contained in:
1 parent
1376b3f44f
commit
d528818c29
1 file changed
+10
@@ -10,6 +10,16 @@ import asyncio
|
|||||||
import logging
|
import logging
|
||||||
import httpx
|
import httpx
|
||||||
import random
|
import random
|
||||||
|
import re
|
||||||
|
import hashlib
|
||||||
|
from datetime import datetime, timedelta
|
||||||
|
from typing import Any, Optional
|
||||||
|
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models import Enseigne, Catalogue, CataloguePage, ScrapingLog
|
||||||
|
from app.services.browserless_service import browserless_service
|
||||||
|
|
||||||
async def _fetch_with_fallback(url: str) -> str:
|
async def _fetch_with_fallback(url: str) -> str:
|
||||||
"""Fetch content using Browserless first, then fallback to HTTPX."""
|
"""Fetch content using Browserless first, then fallback to HTTPX."""
|
||||||
|
|||||||
Reference in new issue
Block a user