feat: add headless Chromium fallback to recipe scraper
Build and Deploy / build-and-deploy (push) Has been cancelled
Build and Deploy / build-and-deploy (push) Has been cancelled
Uses Playwright to drive Chromium when the primary requests-based fetch fails (e.g., Cloudflare, JS-required pages). Hybrid approach: try fast path first, fall back to Chromium only on failure. Dockerfile: add playwright==1.52.0 pip package, run playwright install-deps chromium + playwright install chromium to download the browser and all system libraries. scrape-recipe: add fetch_with_chromium() that tries Playwright first, then chromium --headless --dump-dom as a subprocess fallback. Modify main() to catch primary fetch errors and call Chromium fallback before giving up.
This commit is contained in:
+85
-2
@@ -4,6 +4,10 @@
|
||||
Uses the recipe-scrapers library (https://github.com/hhursev/recipe-scrapers)
|
||||
to download a recipe page and extract the schema.org JSON-LD recipe data.
|
||||
|
||||
Hybrid fetch: tries plain HTTP first (fast path). If that fails (403,
|
||||
Cloudflare, connection error), falls back to headless Chromium via Playwright
|
||||
to get the rendered HTML.
|
||||
|
||||
Outputs the raw schema.org JSON-LD to stdout as a JSON object suitable for
|
||||
parsing by Roux's SchemaOrg Haskell types.
|
||||
|
||||
@@ -18,11 +22,72 @@ Examples:
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from recipe_scrapers import scrape_html
|
||||
|
||||
|
||||
def fetch_with_chromium(url: str, timeout: int = 30) -> str:
|
||||
"""Fetch rendered HTML from a URL using headless Chromium via Playwright.
|
||||
|
||||
Falls back to chromium --headless --dump-dom as a subprocess if Playwright
|
||||
is not available (unlikely in Docker, possible in bare-metal dev).
|
||||
"""
|
||||
html = None
|
||||
|
||||
# Try Playwright first (primary path in Docker)
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(
|
||||
headless=True,
|
||||
args=["--no-sandbox", "--disable-setuid-sandbox"],
|
||||
)
|
||||
page = browser.new_page()
|
||||
page.goto(url, wait_until="networkidle", timeout=timeout * 1000)
|
||||
html = page.content()
|
||||
browser.close()
|
||||
return html
|
||||
except ImportError:
|
||||
pass # Playwright not installed, try subprocess approach
|
||||
except Exception as e:
|
||||
print(f"Playwright failed: {e}", file=sys.stderr)
|
||||
print("Falling back to chromium --headless --dump-dom", file=sys.stderr)
|
||||
|
||||
# Fallback: chromium --headless --dump-dom
|
||||
chromium_path = os.environ.get("CHROMIUM_PATH", "/usr/bin/chromium")
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[chromium_path, "--headless", "--dump-dom", url],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout:
|
||||
return result.stdout
|
||||
stderr_preview = result.stderr[:500] if result.stderr else "(empty)"
|
||||
print(
|
||||
f"chromium --dump-dom failed (exit {result.returncode}): {stderr_preview}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
print(
|
||||
f"chromium not found at '{chromium_path}' and Playwright not available",
|
||||
file=sys.stderr,
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"chromium --dump-dom error: {e}", file=sys.stderr)
|
||||
|
||||
raise RuntimeError(
|
||||
"Could not fetch page with Chromium or Playwright. "
|
||||
"Ensure playwright is installed and chromium is available."
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Extract schema.org JSON-LD from a recipe URL"
|
||||
@@ -46,8 +111,26 @@ def main() -> None:
|
||||
html = f.read()
|
||||
scraper = scrape_html(html=html, org_url=args.url, online=False)
|
||||
else:
|
||||
# Download the page and scrape it
|
||||
scraper = scrape_html(html=None, org_url=args.url, online=True)
|
||||
try:
|
||||
# Fast path: use requests via recipe-scrapers
|
||||
scraper = scrape_html(html=None, org_url=args.url, online=True)
|
||||
except Exception as primary_err:
|
||||
print(
|
||||
f"Primary fetch failed ({type(primary_err).__name__}: {primary_err}), "
|
||||
f"trying Chromium...",
|
||||
file=sys.stderr,
|
||||
)
|
||||
try:
|
||||
html = fetch_with_chromium(args.url)
|
||||
scraper = scrape_html(
|
||||
html=html, org_url=args.url, online=False
|
||||
)
|
||||
except Exception as chrome_err:
|
||||
raise RuntimeError(
|
||||
f"Could not fetch '{args.url}'. "
|
||||
f"Primary error: {primary_err}. "
|
||||
f"Chromium fallback error: {chrome_err}."
|
||||
) from chrome_err
|
||||
|
||||
# Extract the raw schema.org data from the scraper
|
||||
schema = getattr(scraper, "schema", None)
|
||||
|
||||
Reference in New Issue
Block a user