From ddb8577c4bff01b8c02f76aa8f2f0b70f4bdc2f9 Mon Sep 17 00:00:00 2001 From: James Brechtel Date: Wed, 20 May 2026 22:44:15 -0400 Subject: [PATCH] feat: add headless Chromium fallback to recipe scraper Uses Playwright to drive Chromium when the primary requests-based fetch fails (e.g., Cloudflare, JS-required pages). Hybrid approach: try fast path first, fall back to Chromium only on failure. Dockerfile: add playwright==1.52.0 pip package, run playwright install-deps chromium + playwright install chromium to download the browser and all system libraries. scrape-recipe: add fetch_with_chromium() that tries Playwright first, then chromium --headless --dump-dom as a subprocess fallback. Modify main() to catch primary fetch errors and call Chromium fallback before giving up. --- Dockerfile | 8 +- .../2026-05-20-headless-chromium-scraper.md | 258 ++++++++++++++++++ scripts/scrape-recipe | 87 +++++- 3 files changed, 349 insertions(+), 4 deletions(-) create mode 100644 docs/plans/2026-05-20-headless-chromium-scraper.md diff --git a/Dockerfile b/Dockerfile index 2eab883..bdabbcd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,11 +4,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates curl python3 python3-pip python3-venv tini && \ rm -rf /var/lib/apt/lists/* -# Install recipe-scrapers in a virtualenv to isolate dependencies. +# Install recipe-scrapers and Playwright in a virtualenv to isolate dependencies. RUN python3 -m venv /opt/roux-venv && \ - /opt/roux-venv/bin/pip install --no-cache-dir recipe-scrapers==15.11.0 && \ + /opt/roux-venv/bin/pip install --no-cache-dir recipe-scrapers==15.11.0 playwright==1.52.0 && \ rm -rf /root/.cache/pip +# Install Playwright system dependencies and Chromium browser. +RUN /opt/roux-venv/bin/playwright install-deps chromium && \ + /opt/roux-venv/bin/playwright install chromium + # Build timestamp — read at runtime for diagnostics. RUN mkdir -p /build && date -u '+%Y-%m-%d %H:%M UTC' > /build/build-time diff --git a/docs/plans/2026-05-20-headless-chromium-scraper.md b/docs/plans/2026-05-20-headless-chromium-scraper.md new file mode 100644 index 0000000..0d45826 --- /dev/null +++ b/docs/plans/2026-05-20-headless-chromium-scraper.md @@ -0,0 +1,258 @@ +# Headless Chromium Scraper Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add a Chromium-based fallback to the recipe scraper so sites that block plain HTTP requests (Cloudflare, JS-required pages, etc.) can still be imported. + +**Architecture:** Hybrid approach — try the fast `requests` path first (current behavior). If it fails with an HTTP error or connection error, fall back to Playwright-driven headless Chromium to fetch the rendered HTML, then pass it to the existing schema.org extraction pipeline. No Haskell-side changes needed. + +**Tech Stack:** Python, Playwright, Chromium, recipe-scrapers (existing), Docker (debian:bookworm-slim) + +--- + +## Files to change + +| File | Change | Responsibility | +|------|--------|---------------| +| `Dockerfile` | Add Chromium + Playwright to the image | Browser rendering | +| `scripts/scrape-recipe` | Add hybrid fetch: try `requests` first, fall back to Playwright | Fetch + extraction | +| `docker-compose.yml` | No changes needed (same image) | — | +| `src/Roux/Server.hs` | No changes needed (calls script same way) | — | + +--- + +## Task 1: Add Chromium + Playwright to Dockerfile + +**Files:** +- Modify: `Dockerfile` + +**Reasoning:** The current image only has `recipe-scrapers` in a venv. We need: +1. `libffi-dev` and other Playwright system deps (`playwright install chromium` downloads its own Chromium) +2. `playwright` Python package in the venv +3. Chromium browser downloaded by `playwright install chromium` +4. `--no-sandbox` flag support (common in Docker environments) + +- [ ] **Step 1: Add Playwright deps to apt-get install** + +Add `libffi-dev libnss3 libnspr4 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libdbus-1-3 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 libpango-1.0-0 libcairo2 libasound2` to the `apt-get install` line. + +- [ ] **Step 2: Install Playwright in the venv** + +Add `playwright==1.52.0` to the pip install line (same line as `recipe-scrapers`). + +- [ ] **Step 3: Install Chromium browser** + +Add a RUN step after pip install: +```dockerfile +RUN /opt/roux-venv/bin/playwright install chromium +``` + +This downloads Chromium (~200MB) to `/root/.cache/ms-playwright/`. + +- [ ] **Step 4: Add CHROMIUM_PATH env var** + +Add: +```dockerfile +ENV CHROMIUM_PATH="/root/.cache/ms-playwright/chromium-1161/chrome-linux/chrome" +``` + +This lets the Python script locate the browser without relying on Playwright's discovery (which can fail in Docker). + +- [ ] **Step 5: Build and verify** + +```bash +cd /work/personal/roux/roux-main +docker build -t roux-test . +``` + +Expected: Build succeeds, Chromium is downloaded. + +--- + +## Task 2: Modify scrape-recipe to add Chromium fallback + +**Files:** +- Modify: `scripts/scrape-recipe` + +**Reasoning:** The current script calls `scrape_html(html=None, org_url=url, online=True)` which uses `requests` under the hood. We wrap this in a try/except. On failure, we use Playwright to launch Chromium headless, get the rendered HTML, and feed it to `scrape_html(html=html, org_url=url, online=False)`. + +Key design decisions: +- Use Playwright's sync API (`sync_playwright`) — simpler for a CLI script +- Pass `--no-sandbox` to Chromium (required in Docker) +- Set a 30-second timeout for page navigation +- Extract `` tag content from the fully rendered page +- Only fall back if the primary request fails (exception from `requests`) + +- [ ] **Step 1: Add fallback import at the top** + +Add after the existing imports: +```python +import os +import subprocess +import tempfile +``` + +No need to import Playwright at module level — only import it inside the fallback path so the script works even if Playwright isn't installed (for local dev without Docker). + +- [ ] **Step 2: Add the Chromium fallback function** + +Add before `main()`: +```python +def fetch_with_chromium(url: str, timeout: int = 30) -> str: + """Fetch rendered HTML from a URL using headless Chromium via Playwright. + + Falls back to chromium --headless --dump-dom as a subprocess if Playwright + is not available (unlikely in Docker, possible in bare-metal dev). + """ + html = None + + # Try Playwright first (primary path in Docker) + try: + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + browser = p.chromium.launch( + headless=True, + args=["--no-sandbox", "--disable-setuid-sandbox"] + ) + page = browser.new_page() + page.goto(url, wait_until="networkidle", timeout=timeout * 1000) + html = page.content() + browser.close() + return html + except ImportError: + pass # Playwright not installed, try subprocess approach + except Exception as e: + print(f"Playwright failed: {e}", file=sys.stderr) + print("Falling back to chromium --headless --dump-dom", file=sys.stderr) + + # Fallback: chromium --headless --dump-dom + chromium_path = os.environ.get( + "CHROMIUM_PATH", + "/usr/bin/chromium" + ) + try: + result = subprocess.run( + [chromium_path, "--headless", "--dump-dom", url], + capture_output=True, + text=True, + timeout=timeout + ) + if result.returncode == 0 and result.stdout: + return result.stdout + print(f"chromium --dump-dom failed: {result.stderr[:500]}", file=sys.stderr) + except FileNotFoundError: + pass + except Exception as e: + print(f"chromium --dump-dom error: {e}", file=sys.stderr) + + raise RuntimeError("Could not fetch page with Chromium or Playwright") +``` + +- [ ] **Step 3: Modify the main fetch logic** + +Replace the current download block: +```python +if args.html: + with open(args.html) as f: + html = f.read() + scraper = scrape_html(html=html, org_url=args.url, online=False) +else: + # Download the page and scrape it + scraper = scrape_html(html=None, org_url=args.url, online=True) +``` + +With hybrid logic: +```python +if args.html: + with open(args.html) as f: + html = f.read() + scraper = scrape_html(html=html, org_url=args.url, online=False) +else: + try: + # Fast path: use requests via recipe-scrapers + scraper = scrape_html(html=None, org_url=args.url, online=True) + except Exception as primary_err: + print(f"Primary fetch failed ({type(primary_err).__name__}: {primary_err}), trying Chromium...", file=sys.stderr) + try: + html = fetch_with_chromium(args.url) + scraper = scrape_html(html=html, org_url=args.url, online=False) + except Exception as chrome_err: + print(f"Chromium fallback also failed: {chrome_err}", file=sys.stderr) + raise RuntimeError( + f"Could not fetch '{args.url}'. " + f"Primary error: {primary_err}. " + f"Chromium fallback error: {chrome_err}." + ) from chrome_err +``` + +- [ ] **Step 4: Verify the script works with a known site** + +```bash +cd /work/personal/roux/roux-main +docker build -t roux-test . +docker run --rm roux-test /opt/roux-venv/bin/python /usr/local/bin/scrape-recipe --pretty https://cooking.nytimes.com/recipes/12177-fried-rice +``` + +Expected: outputs JSON-LD schema.org data. + +- [ ] **Step 5: Verify the fallback path** + +Test that the fallback actually triggers by pointing at a URL that blocks requests but works in a browser. Or simulate by temporarily modifying the script: + +```bash +docker run --rm roux-test /opt/roux-venv/bin/python -c " +from playwright.sync_api import sync_playwright +with sync_playwright() as p: + b = p.chromium.launch(headless=True, args=['--no-sandbox']) + page = b.new_page() + page.goto('https://cooking.nytimes.com/recipes/12177-fried-rice', wait_until='networkidle') + print(page.title()) + b.close() +" +``` + +Expected: "Fried Rice Recipe" + +- [ ] **Step 6: Commit** + +```bash +git add Dockerfile scripts/scrape-recipe +git commit -m "feat: add headless Chromium fallback to recipe scraper + +Uses Playwright to drive Chromium when the primary requests-based +fetch fails (e.g., Cloudflare, JS-required pages). Hybrid approach: +try fast path first, fall back to Chromium only on failure. +No Haskell-side changes needed." +``` + +--- + +## Task 3: Verify end-to-end import + +**Files:** +- No file changes — verification only + +- [ ] **Step 1: Build the Docker image** + +```bash +docker build -t roux-test . +``` + +- [ ] **Step 2: Run the scraper directly** + +```bash +# Fast path (should succeed with NYT) +docker run --rm roux-test /opt/roux-venv/bin/python /usr/local/bin/scrape-recipe --pretty https://cooking.nytimes.com/recipes/12177-fried-rice 2>&1 | head -5 + +# Verify the JSON output starts with a schema.org type +docker run --rm roux-test /opt/roux-venv/bin/python /usr/local/bin/scrape-recipe --pretty https://cooking.nytimes.com/recipes/12177-fried-rice 2>&1 | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('@type','no type'))" +``` + +Expected for both: valid JSON with `@type: "Recipe"` or similar. + +- [ ] **Step 3: Push to origin** + +```bash +git push origin main +``` diff --git a/scripts/scrape-recipe b/scripts/scrape-recipe index 4e84547..40e5761 100755 --- a/scripts/scrape-recipe +++ b/scripts/scrape-recipe @@ -4,6 +4,10 @@ Uses the recipe-scrapers library (https://github.com/hhursev/recipe-scrapers) to download a recipe page and extract the schema.org JSON-LD recipe data. +Hybrid fetch: tries plain HTTP first (fast path). If that fails (403, +Cloudflare, connection error), falls back to headless Chromium via Playwright +to get the rendered HTML. + Outputs the raw schema.org JSON-LD to stdout as a JSON object suitable for parsing by Roux's SchemaOrg Haskell types. @@ -18,11 +22,72 @@ Examples: import argparse import json +import os +import subprocess import sys +import tempfile from recipe_scrapers import scrape_html +def fetch_with_chromium(url: str, timeout: int = 30) -> str: + """Fetch rendered HTML from a URL using headless Chromium via Playwright. + + Falls back to chromium --headless --dump-dom as a subprocess if Playwright + is not available (unlikely in Docker, possible in bare-metal dev). + """ + html = None + + # Try Playwright first (primary path in Docker) + try: + from playwright.sync_api import sync_playwright + + with sync_playwright() as p: + browser = p.chromium.launch( + headless=True, + args=["--no-sandbox", "--disable-setuid-sandbox"], + ) + page = browser.new_page() + page.goto(url, wait_until="networkidle", timeout=timeout * 1000) + html = page.content() + browser.close() + return html + except ImportError: + pass # Playwright not installed, try subprocess approach + except Exception as e: + print(f"Playwright failed: {e}", file=sys.stderr) + print("Falling back to chromium --headless --dump-dom", file=sys.stderr) + + # Fallback: chromium --headless --dump-dom + chromium_path = os.environ.get("CHROMIUM_PATH", "/usr/bin/chromium") + try: + result = subprocess.run( + [chromium_path, "--headless", "--dump-dom", url], + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode == 0 and result.stdout: + return result.stdout + stderr_preview = result.stderr[:500] if result.stderr else "(empty)" + print( + f"chromium --dump-dom failed (exit {result.returncode}): {stderr_preview}", + file=sys.stderr, + ) + except FileNotFoundError: + print( + f"chromium not found at '{chromium_path}' and Playwright not available", + file=sys.stderr, + ) + except Exception as e: + print(f"chromium --dump-dom error: {e}", file=sys.stderr) + + raise RuntimeError( + "Could not fetch page with Chromium or Playwright. " + "Ensure playwright is installed and chromium is available." + ) + + def main() -> None: parser = argparse.ArgumentParser( description="Extract schema.org JSON-LD from a recipe URL" @@ -46,8 +111,26 @@ def main() -> None: html = f.read() scraper = scrape_html(html=html, org_url=args.url, online=False) else: - # Download the page and scrape it - scraper = scrape_html(html=None, org_url=args.url, online=True) + try: + # Fast path: use requests via recipe-scrapers + scraper = scrape_html(html=None, org_url=args.url, online=True) + except Exception as primary_err: + print( + f"Primary fetch failed ({type(primary_err).__name__}: {primary_err}), " + f"trying Chromium...", + file=sys.stderr, + ) + try: + html = fetch_with_chromium(args.url) + scraper = scrape_html( + html=html, org_url=args.url, online=False + ) + except Exception as chrome_err: + raise RuntimeError( + f"Could not fetch '{args.url}'. " + f"Primary error: {primary_err}. " + f"Chromium fallback error: {chrome_err}." + ) from chrome_err # Extract the raw schema.org data from the scraper schema = getattr(scraper, "schema", None)