8a410b0c3c
Rewrite the README for external users, neutralize example identities in code comments, generalize the OIDC group note, and simplify the gitleaks workflow header.
378 lines
15 KiB
Python
378 lines
15 KiB
Python
"""Discover v0.1 — scrape external recipe URLs into the discover corpus.
|
|
|
|
Pipeline per URL:
|
|
1. recipe_scrapers.scrape_me(url) → schema.org structured recipe
|
|
2. Reshape into a Mealie-ish dict (name, description, recipeYield,
|
|
recipeIngredient[{note}], recipeInstructions[{text}])
|
|
3. INSERT IGNORE into cauldron_discovered_recipes (UNIQUE on source_url)
|
|
4. forge.enrich_recipe(reshaped) → Hecate-tier metadata
|
|
5. Persist meta_json, flip status raw → enriched
|
|
|
|
Same daemon-thread + cancel + stuck-recovery pattern as enrich/sterilize.
|
|
|
|
Seed sources are hardcoded URL lists per source_seed (allrecipes-popular,
|
|
bbc-popular, smitten-kitchen-recent, ...). The user supplies a seed name OR
|
|
a literal list of URLs via the admin endpoint. Either way, the runner
|
|
walks the list, scrape→insert→enrich each, and emits progress.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import threading
|
|
from urllib.parse import urlparse
|
|
|
|
import ipaddress as _ipaddr
|
|
import socket as _socket
|
|
|
|
from .db import DB
|
|
from .forge import Forge, ForgeError
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# Curated seed URL lists for v0.1 dogfood. Each is a small starter pack —
|
|
# we expand later by adding sitemap/category-page walkers. Keeping these
|
|
# manual lets v0.1 ship without a separate site-walker per source.
|
|
SEED_URLS: dict[str, list[str]] = {
|
|
"allrecipes-popular": [
|
|
"https://www.allrecipes.com/recipe/24074/alyssas-chicken/",
|
|
"https://www.allrecipes.com/recipe/229960/world-best-now-veggie-burgers/",
|
|
"https://www.allrecipes.com/recipe/16641/old-fashioned-mac-and-cheese/",
|
|
"https://www.allrecipes.com/recipe/8499082/instant-pot-pulled-pork/",
|
|
"https://www.allrecipes.com/recipe/220854/chef-johns-creamy-mushroom-pasta/",
|
|
"https://www.allrecipes.com/recipe/8514308/dr-pepper-pulled-pork/",
|
|
"https://www.allrecipes.com/recipe/16700/salisbury-steak/",
|
|
"https://www.allrecipes.com/recipe/8536048/oven-baked-bbq-chicken-thighs/",
|
|
],
|
|
"bbc-good-food": [
|
|
"https://www.bbcgoodfood.com/recipes/spaghetti-bolognese-recipe",
|
|
"https://www.bbcgoodfood.com/recipes/best-spaghetti-carbonara-recipe",
|
|
"https://www.bbcgoodfood.com/recipes/easy-chicken-curry",
|
|
"https://www.bbcgoodfood.com/recipes/chilli-con-carne-recipe",
|
|
"https://www.bbcgoodfood.com/recipes/perfect-roast-chicken",
|
|
"https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
|
|
"https://www.bbcgoodfood.com/recipes/sticky-toffee-pudding",
|
|
],
|
|
"smitten-kitchen": [
|
|
"https://smittenkitchen.com/2023/02/black-pepper-chicken/",
|
|
"https://smittenkitchen.com/2024/01/orecchiette-with-broccoli-rabe/",
|
|
"https://smittenkitchen.com/2023/09/baked-orzo-with-eggplant-and-mozzarella/",
|
|
"https://smittenkitchen.com/2022/12/cacio-e-pepe-soup-with-broccoli-rabe/",
|
|
"https://smittenkitchen.com/2022/05/spinach-chickpea-skillet/",
|
|
],
|
|
"pinch-of-yum": [
|
|
"https://pinchofyum.com/the-best-soft-chocolate-chip-cookies",
|
|
"https://pinchofyum.com/spicy-peanut-soba-noodle-salad",
|
|
"https://pinchofyum.com/best-chicken-marinade",
|
|
"https://pinchofyum.com/15-minute-meal-prep-cilantro-lime-chicken-and-cauliflower-rice",
|
|
"https://pinchofyum.com/pesto-cavatappi",
|
|
],
|
|
"half-baked-harvest": [
|
|
"https://www.halfbakedharvest.com/cajun-chicken-pasta/",
|
|
"https://www.halfbakedharvest.com/garlic-butter-creamed-spinach-salmon/",
|
|
"https://www.halfbakedharvest.com/spicy-pretzel-chicken/",
|
|
"https://www.halfbakedharvest.com/crispy-buffalo-chicken-tacos/",
|
|
"https://www.halfbakedharvest.com/butter-chicken-meatballs/",
|
|
],
|
|
}
|
|
|
|
|
|
def list_seeds() -> list[dict]:
|
|
"""For the /discover admin UI: name + count of curated URLs per seed."""
|
|
return [{"name": k, "count": len(v)} for k, v in SEED_URLS.items()]
|
|
|
|
|
|
def is_public_url(url: str) -> tuple[bool, str]:
|
|
"""SSRF guard: validate that `url` resolves to a non-private,
|
|
non-loopback, non-link-local IP. Returns (ok, reason).
|
|
|
|
Used by both `/api/discover/scrape-start` (pre-queue rejection) and
|
|
`_scrape_one` (defense-in-depth before fetch). Without this, any
|
|
session user could queue URLs pointing at the LAN, the docker
|
|
bridge, or cloud metadata endpoints (169.254.169.254, etc).
|
|
|
|
Strategy:
|
|
1. Parse the URL. Reject non-http(s) schemes.
|
|
2. Extract hostname. Reject if IP-literal pointing at private space.
|
|
3. Resolve via getaddrinfo (covers IPv4 + IPv6 + IDNs).
|
|
4. For each resolved address, reject if .is_private, .is_loopback,
|
|
.is_link_local, .is_multicast, .is_reserved, .is_unspecified.
|
|
|
|
Note: this is best-effort. A malicious resolver could DNS-rebind
|
|
between this check and the actual GET. recipe-scrapers also makes
|
|
its own HTTP calls in scrape_me — those bypass this guard. Acceptable
|
|
for v0.1 (LAN-only deployment, family OIDC). The right answer
|
|
long-term is a custom requests transport that re-validates per-
|
|
connection like Mealie's pkgs/safehttp."""
|
|
from urllib.parse import urlparse
|
|
try:
|
|
parsed = urlparse(url)
|
|
except Exception as e:
|
|
return (False, f"unparseable url: {e}")
|
|
if parsed.scheme not in ("http", "https"):
|
|
return (False, f"scheme not allowed: {parsed.scheme!r}")
|
|
host = parsed.hostname
|
|
if not host:
|
|
return (False, "no hostname in url")
|
|
# Reject IP-literals pointing into private / loopback / link-local /
|
|
# multicast / reserved space directly (saves a DNS roundtrip too).
|
|
try:
|
|
ip = _ipaddr.ip_address(host)
|
|
if (ip.is_private or ip.is_loopback or ip.is_link_local
|
|
or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
|
|
return (False, f"ip in restricted range: {ip}")
|
|
return (True, "")
|
|
except ValueError:
|
|
pass # not an IP literal — resolve via DNS
|
|
|
|
# Resolve and check every returned address.
|
|
try:
|
|
infos = _socket.getaddrinfo(host, None, type=_socket.SOCK_STREAM)
|
|
except _socket.gaierror as e:
|
|
return (False, f"dns resolution failed: {e}")
|
|
for info in infos:
|
|
addr = info[4][0]
|
|
try:
|
|
ip = _ipaddr.ip_address(addr)
|
|
except ValueError:
|
|
continue
|
|
if (ip.is_private or ip.is_loopback or ip.is_link_local
|
|
or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
|
|
return (False, f"resolves to restricted ip: {ip} (host={host})")
|
|
return (True, "")
|
|
|
|
|
|
def _slug_from_url(url: str) -> str | None:
|
|
"""Cheap slug fallback when the scraper doesn't expose one."""
|
|
try:
|
|
parts = [p for p in urlparse(url).path.split("/") if p]
|
|
return parts[-1][:255] if parts else None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _safe_call(fn, default=None):
|
|
"""recipe_scrapers raises various Exception subclasses for missing
|
|
fields. Swallow them per-field rather than aborting the whole scrape."""
|
|
try:
|
|
return fn()
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def _to_mealie_shape(scraper, source_url: str) -> dict:
|
|
"""Reshape a recipe_scrapers.AbstractScraper into the dict shape
|
|
forge.enrich_recipe expects (a Mealie recipe). Falls back gracefully
|
|
when individual fields are unavailable."""
|
|
title = _safe_call(scraper.title) or ""
|
|
description = _safe_call(getattr(scraper, "description", lambda: ""), "") or ""
|
|
yields = _safe_call(scraper.yields, "") or ""
|
|
image = _safe_call(scraper.image, "") or ""
|
|
|
|
ings_raw = _safe_call(scraper.ingredients, []) or []
|
|
ingredients = [
|
|
{"note": str(x).strip()}
|
|
for x in ings_raw
|
|
if x and str(x).strip()
|
|
]
|
|
|
|
# Prefer instructions_list when supported; some scrapers only expose
|
|
# the joined string.
|
|
steps_list: list[str] = []
|
|
instructions_list = _safe_call(getattr(scraper, "instructions_list", lambda: None), None)
|
|
if instructions_list:
|
|
steps_list = [str(s).strip() for s in instructions_list if s and str(s).strip()]
|
|
else:
|
|
joined = _safe_call(scraper.instructions, "") or ""
|
|
steps_list = [s.strip() for s in joined.split("\n") if s.strip()]
|
|
instructions = [{"text": s} for s in steps_list]
|
|
|
|
return {
|
|
"name": title,
|
|
"description": description,
|
|
"recipeYield": yields,
|
|
"image": image,
|
|
"source_url": source_url,
|
|
"recipeIngredient": ingredients,
|
|
"recipeInstructions": instructions,
|
|
}
|
|
|
|
|
|
def _scrape_one(url: str) -> tuple[dict, str | None] | None:
|
|
"""Scrape a single URL. Returns (mealie_shape_dict, image_url) on
|
|
success. Returns None on any unrecoverable scraper error."""
|
|
# SSRF defense-in-depth: even though /api/discover/scrape-start
|
|
# validates URLs at queue time, re-check here so any future caller
|
|
# (cron, admin script, future bulk runner) can't bypass it.
|
|
ok, reason = is_public_url(url)
|
|
if not ok:
|
|
log.warning("[discover] refusing private/restricted url %s (%s)", url, reason)
|
|
return None
|
|
|
|
try:
|
|
from recipe_scrapers import scrape_me # type: ignore
|
|
except ImportError:
|
|
log.exception("[discover] recipe_scrapers not installed")
|
|
return None
|
|
|
|
try:
|
|
scraper = scrape_me(url)
|
|
except Exception as e:
|
|
# scraper_exists_for is False or the request failed.
|
|
# Fall back to scrape_html with supported_only=False so unknown
|
|
# sites still get a JSON-LD/microdata pass.
|
|
try:
|
|
from recipe_scrapers import scrape_html # type: ignore
|
|
import requests as _rq
|
|
resp = _rq.get(
|
|
url,
|
|
timeout=15,
|
|
# allow_redirects=False: is_public_url validated the
|
|
# original host as public; a 30x to 127.0.0.1 / 169.254.x
|
|
# would otherwise route this scrape worker at internal
|
|
# services (LAN scanner, cloud metadata IMDS). Treat 30x
|
|
# as scrape failure rather than chase the redirect chain.
|
|
# The recipe_scrapers primary path has its own internal
|
|
# request chain that's a known residual — the docstring
|
|
# on is_public_url notes the long-term answer is a
|
|
# custom requests transport that re-validates per hop.
|
|
allow_redirects=False,
|
|
headers={
|
|
# Realistic desktop UA — many recipe sites 403 anything
|
|
# that smells like a bot. We're identifying as a normal
|
|
# browser; per-site robots.txt we still respect via
|
|
# recipe_scrapers' built-in wild_mode safety nets.
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/120.0 Safari/537.36"
|
|
),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
},
|
|
)
|
|
if resp.status_code != 200:
|
|
log.warning("[discover] fetch %s -> %s", url, resp.status_code)
|
|
return None
|
|
scraper = scrape_html(
|
|
html=resp.text, org_url=url, supported_only=False
|
|
)
|
|
except Exception as e2:
|
|
log.warning("[discover] scrape_me(%s) failed: %s / fallback: %s", url, e, e2)
|
|
return None
|
|
|
|
shaped = _to_mealie_shape(scraper, url)
|
|
image = shaped.get("image") or None
|
|
if not shaped.get("name"):
|
|
log.warning("[discover] no name extracted from %s", url)
|
|
return None
|
|
if not shaped.get("recipeIngredient"):
|
|
log.warning("[discover] no ingredients extracted from %s", url)
|
|
return None
|
|
return shaped, image
|
|
|
|
|
|
def run_discover(
|
|
*,
|
|
db: DB,
|
|
job_id: int,
|
|
forge: Forge,
|
|
urls: list[str],
|
|
) -> None:
|
|
"""Walk a list of URLs: scrape → insert → enrich. Runs in a daemon
|
|
thread; respects external cancel via state poll."""
|
|
log.info("[discover:%s] start (%d urls)", job_id, len(urls))
|
|
|
|
def _cancelled() -> bool:
|
|
s = db.get_discover_job_state(job_id)
|
|
return s in ("cancelled", "failed", "done")
|
|
|
|
try:
|
|
for url in urls:
|
|
if _cancelled():
|
|
log.info("[discover:%s] aborted (state changed)", job_id)
|
|
return
|
|
|
|
db.update_discover_job_progress(job_id, pages_delta=1)
|
|
|
|
scraped = _scrape_one(url)
|
|
if scraped is None:
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"scrape failed: {url[:200]}"
|
|
)
|
|
continue
|
|
shaped, image = scraped
|
|
|
|
try:
|
|
slug = _slug_from_url(url)
|
|
discover_id = db.insert_discovered_recipe(
|
|
slug=slug,
|
|
source_url=url,
|
|
name=shaped.get("name") or None,
|
|
description=(shaped.get("description") or "")[:60000] or None,
|
|
image_url=image,
|
|
scraped_json=json.dumps(shaped, ensure_ascii=False),
|
|
)
|
|
except Exception as e:
|
|
log.warning("[discover:%s] insert(%s) failed: %s", job_id, url, e)
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"insert: {str(e)[:200]}"
|
|
)
|
|
continue
|
|
|
|
if not discover_id:
|
|
# UNIQUE conflict — already in the corpus from a prior scrape
|
|
db.update_discover_job_progress(job_id, skipped_delta=1)
|
|
continue
|
|
|
|
try:
|
|
meta = forge.enrich_recipe(shaped)
|
|
except (ForgeError, RuntimeError) as e:
|
|
msg = str(e)[:500]
|
|
log.warning("[discover:%s] enrich(%s): %s", job_id, url, msg)
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"enrich: {msg[:200]}"
|
|
)
|
|
# Leave the row in 'raw' so we can retry enrichment later.
|
|
# The recipe IS in the corpus; just hasn't been classified.
|
|
continue
|
|
|
|
try:
|
|
db.update_discovered_meta(
|
|
discover_id,
|
|
meta_json=json.dumps(meta, ensure_ascii=False),
|
|
version=DB.ENRICH_VERSION,
|
|
)
|
|
db.update_discover_job_progress(job_id, added_delta=1)
|
|
except Exception as e:
|
|
log.warning("[discover:%s] persist meta(%s): %s", job_id, url, e)
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"persist: {str(e)[:200]}"
|
|
)
|
|
|
|
db.finalize_discover_job(job_id, state="done")
|
|
log.info("[discover:%s] done", job_id)
|
|
except Exception:
|
|
log.exception("[discover:%s] crashed", job_id)
|
|
try:
|
|
db.finalize_discover_job(job_id, state="failed")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def spawn_thread(
|
|
*,
|
|
db: DB,
|
|
job_id: int,
|
|
forge: Forge,
|
|
urls: list[str],
|
|
) -> threading.Thread:
|
|
t = threading.Thread(
|
|
target=run_discover,
|
|
kwargs={"db": db, "job_id": job_id, "forge": forge, "urls": urls},
|
|
name=f"discover-recipes-{job_id}",
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
return t
|