roux/scripts/scrape-recipe

#!/usr/bin/env python3
"""Extract recipe schema.org JSON-LD from a recipe URL.

Uses the recipe-scrapers library (https://github.com/hhursev/recipe-scrapers)
to download a recipe page and extract the schema.org JSON-LD recipe data.

Hybrid fetch: tries plain HTTP first (fast path). If that fails (403,
Cloudflare, connection error), falls back to headless Chromium via Playwright
to get the rendered HTML.

Outputs the raw schema.org JSON-LD to stdout as a JSON object suitable for
parsing by Roux's SchemaOrg Haskell types.

Usage:
    ./scripts/scrape-recipe <url>
    ./scripts/scrape-recipe --html <file> <url>   # Use pre-fetched HTML

Examples:
    ./scripts/scrape-recipe https://cooking.nytimes.com/recipes/12177-fried-rice
    ./scripts/scrape-recipe --html fried-rice.html https://example.com/recipe
"""

import argparse
import json
import os
import subprocess
import sys
import tempfile

from recipe_scrapers import scrape_html


def fetch_with_chromium(url: str, timeout: int = 30) -> str:
    """Fetch rendered HTML from a URL using headless Chromium via Playwright.

    Falls back to chromium --headless --dump-dom as a subprocess if Playwright
    is not available (unlikely in Docker, possible in bare-metal dev).
    """
    html = None

    # Try Playwright first (primary path in Docker)
    try:
        from playwright.sync_api import sync_playwright

        with sync_playwright() as p:
            browser = p.chromium.launch(
                headless=True,
                args=["--no-sandbox", "--disable-setuid-sandbox"],
            )
            page = browser.new_page()
            page.goto(url, wait_until="networkidle", timeout=timeout * 1000)
            html = page.content()
            browser.close()
        return html
    except ImportError:
        pass  # Playwright not installed, try subprocess approach
    except Exception as e:
        print(f"Playwright failed: {e}", file=sys.stderr)
        print("Falling back to chromium --headless --dump-dom", file=sys.stderr)

    # Fallback: chromium --headless --dump-dom
    chromium_path = os.environ.get("CHROMIUM_PATH", "/usr/bin/chromium")
    try:
        result = subprocess.run(
            [chromium_path, "--headless", "--dump-dom", url],
            capture_output=True,
            text=True,
            timeout=timeout,
        )
        if result.returncode == 0 and result.stdout:
            return result.stdout
        stderr_preview = result.stderr[:500] if result.stderr else "(empty)"
        print(
            f"chromium --dump-dom failed (exit {result.returncode}): {stderr_preview}",
            file=sys.stderr,
        )
    except FileNotFoundError:
        print(
            f"chromium not found at '{chromium_path}' and Playwright not available",
            file=sys.stderr,
        )
    except Exception as e:
        print(f"chromium --dump-dom error: {e}", file=sys.stderr)

    raise RuntimeError(
        "Could not fetch page with Chromium or Playwright. "
        "Ensure playwright is installed and chromium is available."
    )


def main() -> None:
    parser = argparse.ArgumentParser(
        description="Extract schema.org JSON-LD from a recipe URL"
    )
    parser.add_argument("url", help="URL of the recipe")
    parser.add_argument(
        "--html",
        metavar="FILE",
        help="Read HTML from FILE instead of downloading (for testing)",
    )
    parser.add_argument(
        "--pretty",
        action="store_true",
        help="Pretty-print the JSON output",
    )
    args = parser.parse_args()

    try:
        if args.html:
            with open(args.html) as f:
                html = f.read()
            scraper = scrape_html(html=html, org_url=args.url, online=False)
        else:
            try:
                # Fast path: use requests via recipe-scrapers
                scraper = scrape_html(html=None, org_url=args.url, online=True)
            except Exception as primary_err:
                print(
                    f"Primary fetch failed ({type(primary_err).__name__}: {primary_err}), "
                    f"trying Chromium...",
                    file=sys.stderr,
                )
                try:
                    html = fetch_with_chromium(args.url)
                    scraper = scrape_html(
                        html=html, org_url=args.url, online=False
                    )
                except Exception as chrome_err:
                    raise RuntimeError(
                        f"Could not fetch '{args.url}'. "
                        f"Primary error: {primary_err}. "
                        f"Chromium fallback error: {chrome_err}."
                    ) from chrome_err

        # Extract the raw schema.org data from the scraper
        schema = getattr(scraper, "schema", None)
        if schema is None:
            print("Error: scraper has no schema data", file=sys.stderr)
            sys.exit(1)

        data: dict = schema.data
        if not data:
            print("Error: schema data is empty", file=sys.stderr)
            sys.exit(1)

        indent = 2 if args.pretty else None
        json.dump(data, sys.stdout, indent=indent, ensure_ascii=False)
        sys.stdout.write("\n")

    except Exception as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)


if __name__ == "__main__":
    main()