#!/usr/bin/env python3 """Extract recipe schema.org JSON-LD from a recipe URL. Uses the recipe-scrapers library (https://github.com/hhursev/recipe-scrapers) to download a recipe page and extract the schema.org JSON-LD recipe data. Hybrid fetch: tries plain HTTP first (fast path). If that fails (403, Cloudflare, connection error), falls back to headless Chromium via Playwright to get the rendered HTML. Outputs the raw schema.org JSON-LD to stdout as a JSON object suitable for parsing by Roux's SchemaOrg Haskell types. Usage: ./scripts/scrape-recipe ./scripts/scrape-recipe --html # Use pre-fetched HTML Examples: ./scripts/scrape-recipe https://cooking.nytimes.com/recipes/12177-fried-rice ./scripts/scrape-recipe --html fried-rice.html https://example.com/recipe """ import argparse import json import os import subprocess import sys import tempfile from recipe_scrapers import scrape_html def fetch_with_chromium(url: str, timeout: int = 30) -> str: """Fetch rendered HTML from a URL using headless Chromium via Playwright. Falls back to chromium --headless --dump-dom as a subprocess if Playwright is not available (unlikely in Docker, possible in bare-metal dev). """ html = None # Try Playwright first (primary path in Docker) try: from playwright.sync_api import sync_playwright with sync_playwright() as p: browser = p.chromium.launch( headless=True, args=["--no-sandbox", "--disable-setuid-sandbox"], ) page = browser.new_page() page.goto(url, wait_until="networkidle", timeout=timeout * 1000) html = page.content() browser.close() return html except ImportError: pass # Playwright not installed, try subprocess approach except Exception as e: print(f"Playwright failed: {e}", file=sys.stderr) print("Falling back to chromium --headless --dump-dom", file=sys.stderr) # Fallback: chromium --headless --dump-dom chromium_path = os.environ.get("CHROMIUM_PATH", "/usr/bin/chromium") try: result = subprocess.run( [chromium_path, "--headless", "--dump-dom", url], capture_output=True, text=True, timeout=timeout, ) if result.returncode == 0 and result.stdout: return result.stdout stderr_preview = result.stderr[:500] if result.stderr else "(empty)" print( f"chromium --dump-dom failed (exit {result.returncode}): {stderr_preview}", file=sys.stderr, ) except FileNotFoundError: print( f"chromium not found at '{chromium_path}' and Playwright not available", file=sys.stderr, ) except Exception as e: print(f"chromium --dump-dom error: {e}", file=sys.stderr) raise RuntimeError( "Could not fetch page with Chromium or Playwright. " "Ensure playwright is installed and chromium is available." ) def main() -> None: parser = argparse.ArgumentParser( description="Extract schema.org JSON-LD from a recipe URL" ) parser.add_argument("url", help="URL of the recipe") parser.add_argument( "--html", metavar="FILE", help="Read HTML from FILE instead of downloading (for testing)", ) parser.add_argument( "--pretty", action="store_true", help="Pretty-print the JSON output", ) args = parser.parse_args() try: if args.html: with open(args.html) as f: html = f.read() scraper = scrape_html(html=html, org_url=args.url, online=False) else: try: # Fast path: use requests via recipe-scrapers scraper = scrape_html(html=None, org_url=args.url, online=True) except Exception as primary_err: print( f"Primary fetch failed ({type(primary_err).__name__}: {primary_err}), " f"trying Chromium...", file=sys.stderr, ) try: html = fetch_with_chromium(args.url) scraper = scrape_html( html=html, org_url=args.url, online=False ) except Exception as chrome_err: raise RuntimeError( f"Could not fetch '{args.url}'. " f"Primary error: {primary_err}. " f"Chromium fallback error: {chrome_err}." ) from chrome_err # Extract the raw schema.org data from the scraper schema = getattr(scraper, "schema", None) if schema is None: print("Error: scraper has no schema data", file=sys.stderr) sys.exit(1) data: dict = schema.data if not data: print("Error: schema data is empty", file=sys.stderr) sys.exit(1) indent = 2 if args.pretty else None json.dump(data, sys.stdout, indent=indent, ensure_ascii=False) sys.stdout.write("\n") except Exception as e: print(f"Error: {e}", file=sys.stderr) sys.exit(1) if __name__ == "__main__": main()