feat: add headless Chromium fallback to recipe scraper
Build and Deploy / build-and-deploy (push) Has been cancelled

Uses Playwright to drive Chromium when the primary requests-based
fetch fails (e.g., Cloudflare, JS-required pages). Hybrid approach:
try fast path first, fall back to Chromium only on failure.

Dockerfile: add playwright==1.52.0 pip package, run playwright
install-deps chromium + playwright install chromium to download
the browser and all system libraries.

scrape-recipe: add fetch_with_chromium() that tries Playwright
first, then chromium --headless --dump-dom as a subprocess fallback.
Modify main() to catch primary fetch errors and call Chromium
fallback before giving up.
This commit is contained in:
2026-05-20 22:44:15 -04:00
parent 8807ba3851
commit ddb8577c4b
3 changed files with 349 additions and 4 deletions
+85 -2
View File
@@ -4,6 +4,10 @@
Uses the recipe-scrapers library (https://github.com/hhursev/recipe-scrapers)
to download a recipe page and extract the schema.org JSON-LD recipe data.
Hybrid fetch: tries plain HTTP first (fast path). If that fails (403,
Cloudflare, connection error), falls back to headless Chromium via Playwright
to get the rendered HTML.
Outputs the raw schema.org JSON-LD to stdout as a JSON object suitable for
parsing by Roux's SchemaOrg Haskell types.
@@ -18,11 +22,72 @@ Examples:
import argparse
import json
import os
import subprocess
import sys
import tempfile
from recipe_scrapers import scrape_html
def fetch_with_chromium(url: str, timeout: int = 30) -> str:
"""Fetch rendered HTML from a URL using headless Chromium via Playwright.
Falls back to chromium --headless --dump-dom as a subprocess if Playwright
is not available (unlikely in Docker, possible in bare-metal dev).
"""
html = None
# Try Playwright first (primary path in Docker)
try:
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(
headless=True,
args=["--no-sandbox", "--disable-setuid-sandbox"],
)
page = browser.new_page()
page.goto(url, wait_until="networkidle", timeout=timeout * 1000)
html = page.content()
browser.close()
return html
except ImportError:
pass # Playwright not installed, try subprocess approach
except Exception as e:
print(f"Playwright failed: {e}", file=sys.stderr)
print("Falling back to chromium --headless --dump-dom", file=sys.stderr)
# Fallback: chromium --headless --dump-dom
chromium_path = os.environ.get("CHROMIUM_PATH", "/usr/bin/chromium")
try:
result = subprocess.run(
[chromium_path, "--headless", "--dump-dom", url],
capture_output=True,
text=True,
timeout=timeout,
)
if result.returncode == 0 and result.stdout:
return result.stdout
stderr_preview = result.stderr[:500] if result.stderr else "(empty)"
print(
f"chromium --dump-dom failed (exit {result.returncode}): {stderr_preview}",
file=sys.stderr,
)
except FileNotFoundError:
print(
f"chromium not found at '{chromium_path}' and Playwright not available",
file=sys.stderr,
)
except Exception as e:
print(f"chromium --dump-dom error: {e}", file=sys.stderr)
raise RuntimeError(
"Could not fetch page with Chromium or Playwright. "
"Ensure playwright is installed and chromium is available."
)
def main() -> None:
parser = argparse.ArgumentParser(
description="Extract schema.org JSON-LD from a recipe URL"
@@ -46,8 +111,26 @@ def main() -> None:
html = f.read()
scraper = scrape_html(html=html, org_url=args.url, online=False)
else:
# Download the page and scrape it
scraper = scrape_html(html=None, org_url=args.url, online=True)
try:
# Fast path: use requests via recipe-scrapers
scraper = scrape_html(html=None, org_url=args.url, online=True)
except Exception as primary_err:
print(
f"Primary fetch failed ({type(primary_err).__name__}: {primary_err}), "
f"trying Chromium...",
file=sys.stderr,
)
try:
html = fetch_with_chromium(args.url)
scraper = scrape_html(
html=html, org_url=args.url, online=False
)
except Exception as chrome_err:
raise RuntimeError(
f"Could not fetch '{args.url}'. "
f"Primary error: {primary_err}. "
f"Chromium fallback error: {chrome_err}."
) from chrome_err
# Extract the raw schema.org data from the scraper
schema = getattr(scraper, "schema", None)