jbrechtel
ed70f6e3c2
Downloaded recipe images from source URLs (NYT Cooking, Love & Lemons, Food Network, etc.) via og:image scraping and DuckDuckGo search + page scraping for recipes without direct source URLs.
96 lines
3.7 KiB
Python
96 lines
3.7 KiB
Python
#!/tmp/recipe-venv/bin/python3
|
|
"""Find images for remaining 6 recipes."""
|
|
import re, sys, time
|
|
from pathlib import Path
|
|
import requests
|
|
from bs4 import BeautifulSoup
|
|
from ddgs import DDGS
|
|
|
|
WORKDIR = Path("/home/jbrechtel/recipes/.worktrees/recipe-images")
|
|
HEADERS = {"User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"}
|
|
|
|
recipes = [
|
|
("Arugula Pesto Chicken with Pecans & Roasted Butternut.cook", "arugula-pesto-chicken-with-pecans-roasted-butternut", "Arugula Pesto Chicken Pecans Roasted Butternut recipe garnishandgather"),
|
|
("Bang Bang Tofu.cook", "bang-bang-tofu", "Bang Bang Tofu recipe"),
|
|
("Chana Chickpea Salad.cook", "chana-chickpea-salad", "Chana Chickpea Salad recipe"),
|
|
("Gaja's Bulgogi Lettuce Wraps.cook", "gaja-s-bulgogi-lettuce-wraps", "Gaja Bulgogi Lettuce Wraps recipe garnishandgather"),
|
|
("Pecan-Brown Butter Trout.cook", "pecan-brown-butter-trout", "Pecan Brown Butter Trout recipe"),
|
|
("West African Peanut Chicken.cook", "west-african-peanut-chicken", "West African Peanut Chicken recipe"),
|
|
]
|
|
|
|
for file, slug, query in recipes:
|
|
fp = WORKDIR / file
|
|
dest = WORKDIR / "recipe-images" / f"{slug}.webp"
|
|
if dest.exists() and dest.stat().st_size > 10000:
|
|
print(f"[{slug}] Already done")
|
|
continue
|
|
|
|
print(f"[{slug}] Searching: {query}")
|
|
try:
|
|
results = list(DDGS().text(query, max_results=5))
|
|
except Exception as e:
|
|
print(f" Search error: {e}")
|
|
continue
|
|
|
|
found = False
|
|
for r in results:
|
|
url = r.get("href", "")
|
|
if not url or any(s in url for s in ["pinterest","facebook","instagram"]):
|
|
continue
|
|
print(f" Fetching: {url}")
|
|
try:
|
|
resp = requests.get(url, headers=HEADERS, timeout=15)
|
|
resp.raise_for_status()
|
|
except Exception as e:
|
|
print(f" Failed: {e}")
|
|
continue
|
|
|
|
soup = BeautifulSoup(resp.text, "html.parser")
|
|
img_urls = []
|
|
|
|
# og:image
|
|
for meta in soup.find_all("meta"):
|
|
c = meta.get("content", "")
|
|
p = (meta.get("property") or "").lower()
|
|
n = (meta.get("name") or "").lower()
|
|
if ("og:image" in p or "og:image" in n or "twitter:image" in n) and c and "logo" not in c.lower():
|
|
img_urls.append(c)
|
|
|
|
# img tags
|
|
for img in soup.find_all("img"):
|
|
src = img.get("src") or img.get("data-src") or ""
|
|
if src.startswith("http") and "logo" not in src.lower() and "icon" not in src.lower():
|
|
img_urls.append(src)
|
|
|
|
for img_url in img_urls[:5]:
|
|
try:
|
|
r2 = requests.get(img_url, headers=HEADERS, timeout=15)
|
|
r2.raise_for_status()
|
|
ct = r2.headers.get("content-type", "")
|
|
if ct.startswith("image/") and len(r2.content) > 10000:
|
|
dest.write_bytes(r2.content)
|
|
print(f" Downloaded ({len(r2.content)} bytes): {img_url[:60]}")
|
|
found = True
|
|
break
|
|
except:
|
|
pass
|
|
if found:
|
|
break
|
|
time.sleep(0.3)
|
|
|
|
if found:
|
|
# Add image field
|
|
url_val = f"https://roux.roo.lol/recipe-images/{slug}.webp"
|
|
content = fp.read_text(encoding="utf-8")
|
|
if re.search(r"^image:", content, re.MULTILINE):
|
|
content = re.sub(r"^image:.*$", f"image: {url_val}", count=1, flags=re.MULTILINE, string=content)
|
|
else:
|
|
content = re.sub(r"^---$", f"image: {url_val}\n---", count=1, flags=re.MULTILINE, string=content)
|
|
fp.write_text(content, encoding="utf-8")
|
|
print(f" Updated image field")
|
|
else:
|
|
print(f" FAILED")
|
|
time.sleep(0.5)
|
|
|
|
print("\nDone!")
|