recipes/bin/scrape-images.py

#!/usr/bin/env python3
"""Scrape og:image from recipe source URLs, download images, update .cook files."""

import csv
import os
import re
import sys
import time
from pathlib import Path

import requests
from bs4 import BeautifulSoup

WORKDIR = Path("/home/jbrechtel/recipes/.worktrees/recipe-images")
HEADERS = {
    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
}


def get_source_url(filepath: Path) -> str | None:
    """Extract the source URL from a .cook file's YAML frontmatter."""
    content = filepath.read_text(encoding="utf-8")
    match = re.search(r"^source:\s*(.+)$", content, re.MULTILINE)
    if not match:
        return None
    val = match.group(1).strip()
    # Only return actual URLs
    if val.startswith("http://") or val.startswith("https://"):
        return val
    return None


def fetch_og_image(url: str, timeout: int = 20) -> str | None:
    """Fetch a page and extract og:image URL."""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=timeout, allow_redirects=True)
        resp.raise_for_status()
    except requests.RequestException as e:
        print(f"  FETCH FAILED: {e}", file=sys.stderr)
        return None

    soup = BeautifulSoup(resp.text, "html.parser")

    # Try og:image meta tag
    for meta in soup.find_all("meta"):
        if meta.get("property") == "og:image" and meta.get("content"):
            return meta["content"]
        if meta.get("name") == "og:image" and meta.get("content"):
            return meta["content"]

    # Try twitter:image
    for meta in soup.find_all("meta"):
        if meta.get("name") == "twitter:image" and meta.get("content"):
            return meta["content"]

    # Try finding the first large image (common in recipe sites)
    for img in soup.find_all("img"):
        src = img.get("src") or ""
        if "logo" in src.lower() or "avatar" in src.lower() or "icon" in src.lower():
            continue
        if src.startswith("http") and any(kw in src.lower() for kw in ["upload", "wp-content", "images"]):
            return src

    return None


def download_image(url: str, dest: Path) -> bool:
    """Download an image from URL to destination path."""
    try:
        resp = requests.get(url, headers=HEADERS, timeout=30, stream=True)
        resp.raise_for_status()
        dest.write_bytes(resp.content)
        return True
    except requests.RequestException as e:
        print(f"  DOWNLOAD FAILED: {e}", file=sys.stderr)
        return False


def add_image_field(filepath: Path, slug: str):
    """Add or update the image field in a .cook file's YAML frontmatter."""
    url = f"https://roux.roo.lol/recipe-images/{slug}.webp"
    content = filepath.read_text(encoding="utf-8")

    # If image field already exists, replace it
    if re.search(r"^image:", content, re.MULTILINE):
        content = re.sub(r"^image:.*$", f"image: {url}", content, count=1, flags=re.MULTILINE)
    else:
        # Insert before closing ---
        content = re.sub(r"^---$", f"image: {url}\n---", content, count=1, flags=re.MULTILINE)

    filepath.write_text(content, encoding="utf-8")
    print(f"  Updated image field: {filepath.name}", file=sys.stderr)


def process_recipe(filepath: Path, slug: str) -> bool:
    """Process a single recipe: find image, download, update frontmatter."""
    dest = WORKDIR / "recipe-images" / f"{slug}.webp"

    # Skip if already downloaded
    if dest.exists():
        print(f"[{slug}] Image already exists, skipping download", file=sys.stderr)
        add_image_field(filepath, slug)
        return True

    # Try to get source URL and scrape image
    src_url = get_source_url(filepath)
    if src_url:
        print(f"[{slug}] Scraping {src_url}", file=sys.stderr)
        img_url = fetch_og_image(src_url)
        if img_url:
            print(f"  Found: {img_url}", file=sys.stderr)
            if download_image(img_url, dest):
                print(f"  Downloaded to {dest.name}", file=sys.stderr)
                add_image_field(filepath, slug)
                return True
            else:
                print(f"  Download failed", file=sys.stderr)

    return False


def main():
    csv_file = WORKDIR / "docs" / "no-image-has-fm.csv"
    if not csv_file.exists():
        print(f"CSV not found: {csv_file}", file=sys.stderr)
        sys.exit(1)

    recipes = []
    with open(csv_file) as f:
        reader = csv.reader(f, delimiter="|")
        for row in reader:
            if len(row) >= 2:
                recipes.append((row[0].strip(), row[1].strip()))

    print(f"Processing {len(recipes)} recipes...", file=sys.stderr)

    success = []
    failed = []

    for filename, slug in recipes:
        filepath = WORKDIR / filename
        if not filepath.exists():
            print(f"[{slug}] File not found: {filepath}", file=sys.stderr)
            failed.append(slug)
            continue

        ok = process_recipe(filepath, slug)
        if ok:
            success.append(slug)
        else:
            failed.append(slug)

        # Be polite to servers
        time.sleep(0.5)

    print(f"\n=== Results ===", file=sys.stderr)
    print(f"Success: {len(success)}", file=sys.stderr)
    print(f"Failed: {len(failed)}", file=sys.stderr)
    if failed:
        print(f"Failed slugs: {', '.join(failed)}", file=sys.stderr)

    # Write failed slugs to file for retry
    if failed:
        with open(WORKDIR / "docs" / "failed-images.csv", "w") as f:
            for slug in failed:
                f.write(f"{slug}\n")


if __name__ == "__main__":
    main()