feat: add recipe scraping script and update Docker image
Build and Deploy / build-and-deploy (push) Successful in 1m55s
Build and Deploy / build-and-deploy (push) Successful in 1m55s
- scripts/scrape-recipe: Python script using recipe-scrapers to extract schema.org JSON-LD from a recipe URL (or local HTML file for testing) - Dockerfile: add Python3 + recipe-scrapers in virtualenv, install the scrape-recipe script at /usr/local/bin/scrape-recipe
This commit is contained in:
+9
-2
@@ -1,15 +1,22 @@
|
|||||||
FROM debian:bookworm-slim
|
FROM debian:bookworm-slim
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ca-certificates curl tini && \
|
ca-certificates curl python3 python3-pip python3-venv tini && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
# Install recipe-scrapers in a virtualenv to isolate dependencies.
|
||||||
|
RUN python3 -m venv /opt/roux-venv && \
|
||||||
|
/opt/roux-venv/bin/pip install --no-cache-dir recipe-scrapers==15.11.0 && \
|
||||||
|
rm -rf /root/.cache/pip
|
||||||
|
|
||||||
# Build timestamp — read at runtime for diagnostics.
|
# Build timestamp — read at runtime for diagnostics.
|
||||||
RUN mkdir -p /build && date -u '+%Y-%m-%d %H:%M UTC' > /build/build-time
|
RUN mkdir -p /build && date -u '+%Y-%m-%d %H:%M UTC' > /build/build-time
|
||||||
|
|
||||||
ADD roux-server /usr/local/bin/roux-server
|
ADD roux-server /usr/local/bin/roux-server
|
||||||
|
ADD scripts/scrape-recipe /usr/local/bin/scrape-recipe
|
||||||
|
|
||||||
RUN chmod +x /usr/local/bin/roux-server
|
RUN chmod +x /usr/local/bin/roux-server /usr/local/bin/scrape-recipe
|
||||||
|
ENV PATH="/opt/roux-venv/bin:${PATH}"
|
||||||
|
|
||||||
ENTRYPOINT ["/usr/bin/tini", "-s", "--"]
|
ENTRYPOINT ["/usr/bin/tini", "-s", "--"]
|
||||||
|
|
||||||
|
|||||||
Executable
+73
@@ -0,0 +1,73 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Extract recipe schema.org JSON-LD from a recipe URL.
|
||||||
|
|
||||||
|
Uses the recipe-scrapers library (https://github.com/hhursev/recipe-scrapers)
|
||||||
|
to download a recipe page and extract the schema.org JSON-LD recipe data.
|
||||||
|
|
||||||
|
Outputs the raw schema.org JSON-LD to stdout as a JSON object suitable for
|
||||||
|
parsing by Roux's SchemaOrg Haskell types.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
./scripts/scrape-recipe <url>
|
||||||
|
./scripts/scrape-recipe --html <file> <url> # Use pre-fetched HTML
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
./scripts/scrape-recipe https://cooking.nytimes.com/recipes/12177-fried-rice
|
||||||
|
./scripts/scrape-recipe --html fried-rice.html https://example.com/recipe
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from recipe_scrapers import scrape_html
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(
|
||||||
|
description="Extract schema.org JSON-LD from a recipe URL"
|
||||||
|
)
|
||||||
|
parser.add_argument("url", help="URL of the recipe")
|
||||||
|
parser.add_argument(
|
||||||
|
"--html",
|
||||||
|
metavar="FILE",
|
||||||
|
help="Read HTML from FILE instead of downloading (for testing)",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--pretty",
|
||||||
|
action="store_true",
|
||||||
|
help="Pretty-print the JSON output",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
if args.html:
|
||||||
|
with open(args.html) as f:
|
||||||
|
html = f.read()
|
||||||
|
scraper = scrape_html(html=html, org_url=args.url, online=False)
|
||||||
|
else:
|
||||||
|
# Download the page and scrape it
|
||||||
|
scraper = scrape_html(html=None, org_url=args.url, online=True)
|
||||||
|
|
||||||
|
# Extract the raw schema.org data from the scraper
|
||||||
|
schema = getattr(scraper, "schema", None)
|
||||||
|
if schema is None:
|
||||||
|
print("Error: scraper has no schema data", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
data: dict = schema.data
|
||||||
|
if not data:
|
||||||
|
print("Error: schema data is empty", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
indent = 2 if args.pretty else None
|
||||||
|
json.dump(data, sys.stdout, indent=indent, ensure_ascii=False)
|
||||||
|
sys.stdout.write("\n")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error: {e}", file=sys.stderr)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
Reference in New Issue
Block a user