jbrechtel
ed70f6e3c2
Downloaded recipe images from source URLs (NYT Cooking, Love & Lemons, Food Network, etc.) via og:image scraping and DuckDuckGo search + page scraping for recipes without direct source URLs.
149 lines
4.8 KiB
Python
149 lines
4.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Search DuckDuckGo Images for recipe photos, download, update .cook files."""
|
|
|
|
import csv
|
|
import re
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
from duckduckgo_search import DDGS
|
|
|
|
WORKDIR = Path("/home/jbrechtel/recipes/.worktrees/recipe-images")
|
|
HEADERS = {
|
|
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36",
|
|
}
|
|
|
|
|
|
def download_image(url: str, dest: Path) -> bool:
|
|
"""Download an image from URL to destination path."""
|
|
try:
|
|
resp = requests.get(url, headers=HEADERS, timeout=30, stream=True)
|
|
resp.raise_for_status()
|
|
# Validate it's actually an image
|
|
content_type = resp.headers.get("content-type", "")
|
|
if not content_type.startswith("image/"):
|
|
print(f" Not an image (content-type: {content_type})", file=sys.stderr)
|
|
return False
|
|
dest.write_bytes(resp.content)
|
|
return True
|
|
except requests.RequestException as e:
|
|
print(f" Download failed: {e}", file=sys.stderr)
|
|
return False
|
|
|
|
|
|
def add_image_field(filepath: Path, slug: str):
|
|
"""Add or update the image field in a .cook file."""
|
|
url = f"https://roux.roo.lol/recipe-images/{slug}.webp"
|
|
content = filepath.read_text(encoding="utf-8")
|
|
|
|
if re.search(r"^image:", content, re.MULTILINE):
|
|
content = re.sub(r"^image:.*$", f"image: {url}", count=1, flags=re.MULTILINE, string=content)
|
|
else:
|
|
content = re.sub(r"^---$", f"image: {url}\n---", count=1, flags=re.MULTILINE, string=content)
|
|
|
|
filepath.write_text(content, encoding="utf-8")
|
|
print(f" Updated {filepath.name}", file=sys.stderr)
|
|
|
|
|
|
def search_images(query: str, max_results: int = 5):
|
|
"""Search DuckDuckGo Images for a recipe and return image URLs."""
|
|
try:
|
|
results = list(DDGS().images(query, max_results=max_results))
|
|
return [r["image"] for r in results if r.get("image")]
|
|
except Exception as e:
|
|
print(f" Image search error: {e}", file=sys.stderr)
|
|
return []
|
|
|
|
|
|
def get_title(filepath: Path) -> str:
|
|
"""Extract title from .cook frontmatter."""
|
|
content = filepath.read_text(encoding="utf-8")
|
|
match = re.search(r"^title:\s*(.+)$", content, re.MULTILINE)
|
|
if match:
|
|
return match.group(1).strip()
|
|
# Fall back to filename
|
|
name = filepath.stem
|
|
return name.replace("_", " ").replace("-", " ")
|
|
|
|
|
|
def main():
|
|
csv_file = WORKDIR / "docs" / "failed-images.csv"
|
|
# Also check which are actually missing
|
|
existing_csv = WORKDIR / "docs" / "no-image-has-fm.csv"
|
|
|
|
recipes = []
|
|
with open(existing_csv) as f:
|
|
reader = csv.reader(f, delimiter="|")
|
|
for row in reader:
|
|
if len(row) >= 2:
|
|
recipes.append((row[0].strip(), row[1].strip()))
|
|
|
|
# Filter to those without an image file or without image field
|
|
to_process = []
|
|
for filename, slug in recipes:
|
|
filepath = WORKDIR / filename
|
|
dest = WORKDIR / "recipe-images" / f"{slug}.webp"
|
|
has_image_file = dest.exists()
|
|
has_image_field = False
|
|
if filepath.exists():
|
|
content = filepath.read_text(encoding="utf-8")
|
|
has_image_field = bool(re.search(r"^image:", content, re.MULTILINE))
|
|
|
|
if not has_image_file or not has_image_field:
|
|
to_process.append((filename, slug))
|
|
|
|
print(f"Processing {len(to_process)} remaining recipes...", file=sys.stderr)
|
|
|
|
success = []
|
|
failed = []
|
|
|
|
for filename, slug in to_process:
|
|
filepath = WORKDIR / filename
|
|
dest = WORKDIR / "recipe-images" / f"{slug}.webp"
|
|
|
|
if not filepath.exists():
|
|
failed.append(slug)
|
|
continue
|
|
|
|
title = get_title(filepath)
|
|
queries = [
|
|
f"{title} recipe",
|
|
title,
|
|
]
|
|
|
|
found = False
|
|
for query in queries:
|
|
print(f"[{slug}] Searching images: {query}", file=sys.stderr)
|
|
urls = search_images(query)
|
|
for img_url in urls:
|
|
# Skip tiny icons, logos
|
|
if any(kw in img_url.lower() for kw in ["logo", "icon", "avatar", "merriam", "wikipedia"]):
|
|
continue
|
|
print(f" Trying: {img_url[:80]}...", file=sys.stderr)
|
|
if download_image(img_url, dest):
|
|
print(f" Downloaded!", file=sys.stderr)
|
|
add_image_field(filepath, slug)
|
|
found = True
|
|
break
|
|
time.sleep(0.2)
|
|
if found:
|
|
break
|
|
time.sleep(0.5)
|
|
|
|
if found:
|
|
success.append(slug)
|
|
else:
|
|
failed.append(slug)
|
|
|
|
print(f"\n=== Results ===", file=sys.stderr)
|
|
print(f"Success: {len(success)}", file=sys.stderr)
|
|
print(f"Failed: {len(failed)}", file=sys.stderr)
|
|
if failed:
|
|
print(f"Failed: {', '.join(failed)}", file=sys.stderr)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|