9328374cf3
the host bind paths + LAN host pins replaced with env defaults. Repository URLs → git.sulkta.com. Audit-changelog scaffolding stripped from inline comments (technical reasoning preserved). README sheds marketing scaffolding. AI-speak in load-bearing prompts/SOULs left alone — that IS the product.
378 lines
15 KiB
Python
378 lines
15 KiB
Python
"""Discover v0.1 — scrape external recipe URLs into the discover corpus.
|
|
|
|
Pipeline per URL:
|
|
1. recipe_scrapers.scrape_me(url) → schema.org structured recipe
|
|
2. Reshape into a Mealie-ish dict (name, description, recipeYield,
|
|
recipeIngredient[{note}], recipeInstructions[{text}])
|
|
3. INSERT IGNORE into cauldron_discovered_recipes (UNIQUE on source_url)
|
|
4. forge.enrich_recipe(reshaped) → Hecate-tier metadata
|
|
5. Persist meta_json, flip status raw → enriched
|
|
|
|
Same daemon-thread + cancel + stuck-recovery pattern as enrich/sterilize.
|
|
|
|
Seed sources are hardcoded URL lists per source_seed (allrecipes-popular,
|
|
bbc-popular, smitten-kitchen-recent, ...). Alice supplies a seed name OR
|
|
a literal list of URLs via the admin endpoint. Either way, the runner
|
|
walks the list, scrape→insert→enrich each, and emits progress.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import threading
|
|
from urllib.parse import urlparse
|
|
|
|
import ipaddress as _ipaddr
|
|
import socket as _socket
|
|
|
|
from .db import DB
|
|
from .forge import Forge, ForgeError
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
# Curated seed URL lists for v0.1 dogfood. Each is a small starter pack —
|
|
# we expand later by adding sitemap/category-page walkers. Keeping these
|
|
# manual lets v0.1 ship without a separate site-walker per source.
|
|
SEED_URLS: dict[str, list[str]] = {
|
|
"allrecipes-popular": [
|
|
"https://www.allrecipes.com/recipe/24074/alyssas-chicken/",
|
|
"https://www.allrecipes.com/recipe/229960/world-best-now-veggie-burgers/",
|
|
"https://www.allrecipes.com/recipe/16641/old-fashioned-mac-and-cheese/",
|
|
"https://www.allrecipes.com/recipe/8499082/instant-pot-pulled-pork/",
|
|
"https://www.allrecipes.com/recipe/220854/chef-johns-creamy-mushroom-pasta/",
|
|
"https://www.allrecipes.com/recipe/8514308/dr-pepper-pulled-pork/",
|
|
"https://www.allrecipes.com/recipe/16700/salisbury-steak/",
|
|
"https://www.allrecipes.com/recipe/8536048/oven-baked-bbq-chicken-thighs/",
|
|
],
|
|
"bbc-good-food": [
|
|
"https://www.bbcgoodfood.com/recipes/spaghetti-bolognese-recipe",
|
|
"https://www.bbcgoodfood.com/recipes/best-spaghetti-carbonara-recipe",
|
|
"https://www.bbcgoodfood.com/recipes/easy-chicken-curry",
|
|
"https://www.bbcgoodfood.com/recipes/chilli-con-carne-recipe",
|
|
"https://www.bbcgoodfood.com/recipes/perfect-roast-chicken",
|
|
"https://www.bbcgoodfood.com/recipes/chicken-tikka-masala",
|
|
"https://www.bbcgoodfood.com/recipes/sticky-toffee-pudding",
|
|
],
|
|
"smitten-kitchen": [
|
|
"https://smittenkitchen.com/2023/02/black-pepper-chicken/",
|
|
"https://smittenkitchen.com/2024/01/orecchiette-with-broccoli-rabe/",
|
|
"https://smittenkitchen.com/2023/09/baked-orzo-with-eggplant-and-mozzarella/",
|
|
"https://smittenkitchen.com/2022/12/cacio-e-pepe-soup-with-broccoli-rabe/",
|
|
"https://smittenkitchen.com/2022/05/spinach-chickpea-skillet/",
|
|
],
|
|
"pinch-of-yum": [
|
|
"https://pinchofyum.com/the-best-soft-chocolate-chip-cookies",
|
|
"https://pinchofyum.com/spicy-peanut-soba-noodle-salad",
|
|
"https://pinchofyum.com/best-chicken-marinade",
|
|
"https://pinchofyum.com/15-minute-meal-prep-cilantro-lime-chicken-and-cauliflower-rice",
|
|
"https://pinchofyum.com/pesto-cavatappi",
|
|
],
|
|
"half-baked-harvest": [
|
|
"https://www.halfbakedharvest.com/cajun-chicken-pasta/",
|
|
"https://www.halfbakedharvest.com/garlic-butter-creamed-spinach-salmon/",
|
|
"https://www.halfbakedharvest.com/spicy-pretzel-chicken/",
|
|
"https://www.halfbakedharvest.com/crispy-buffalo-chicken-tacos/",
|
|
"https://www.halfbakedharvest.com/butter-chicken-meatballs/",
|
|
],
|
|
}
|
|
|
|
|
|
def list_seeds() -> list[dict]:
|
|
"""For the /discover admin UI: name + count of curated URLs per seed."""
|
|
return [{"name": k, "count": len(v)} for k, v in SEED_URLS.items()]
|
|
|
|
|
|
def is_public_url(url: str) -> tuple[bool, str]:
|
|
"""SSRF guard: validate that `url` resolves to a non-private,
|
|
non-loopback, non-link-local IP. Returns (ok, reason).
|
|
|
|
Used by both `/api/discover/scrape-start` (pre-queue rejection) and
|
|
`_scrape_one` (defense-in-depth before fetch). Without this, any
|
|
session user could queue URLs pointing at the LAN, the docker
|
|
bridge, or cloud metadata endpoints (169.254.169.254, etc).
|
|
|
|
Strategy:
|
|
1. Parse the URL. Reject non-http(s) schemes.
|
|
2. Extract hostname. Reject if IP-literal pointing at private space.
|
|
3. Resolve via getaddrinfo (covers IPv4 + IPv6 + IDNs).
|
|
4. For each resolved address, reject if .is_private, .is_loopback,
|
|
.is_link_local, .is_multicast, .is_reserved, .is_unspecified.
|
|
|
|
Note: this is best-effort. A malicious resolver could DNS-rebind
|
|
between this check and the actual GET. recipe-scrapers also makes
|
|
its own HTTP calls in scrape_me — those bypass this guard. Acceptable
|
|
for v0.1 (LAN-only deployment, family OIDC). The right answer
|
|
long-term is a custom requests transport that re-validates per-
|
|
connection like Mealie's pkgs/safehttp."""
|
|
from urllib.parse import urlparse
|
|
try:
|
|
parsed = urlparse(url)
|
|
except Exception as e:
|
|
return (False, f"unparseable url: {e}")
|
|
if parsed.scheme not in ("http", "https"):
|
|
return (False, f"scheme not allowed: {parsed.scheme!r}")
|
|
host = parsed.hostname
|
|
if not host:
|
|
return (False, "no hostname in url")
|
|
# Reject IP-literals pointing into private / loopback / link-local /
|
|
# multicast / reserved space directly (saves a DNS roundtrip too).
|
|
try:
|
|
ip = _ipaddr.ip_address(host)
|
|
if (ip.is_private or ip.is_loopback or ip.is_link_local
|
|
or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
|
|
return (False, f"ip in restricted range: {ip}")
|
|
return (True, "")
|
|
except ValueError:
|
|
pass # not an IP literal — resolve via DNS
|
|
|
|
# Resolve and check every returned address.
|
|
try:
|
|
infos = _socket.getaddrinfo(host, None, type=_socket.SOCK_STREAM)
|
|
except _socket.gaierror as e:
|
|
return (False, f"dns resolution failed: {e}")
|
|
for info in infos:
|
|
addr = info[4][0]
|
|
try:
|
|
ip = _ipaddr.ip_address(addr)
|
|
except ValueError:
|
|
continue
|
|
if (ip.is_private or ip.is_loopback or ip.is_link_local
|
|
or ip.is_multicast or ip.is_reserved or ip.is_unspecified):
|
|
return (False, f"resolves to restricted ip: {ip} (host={host})")
|
|
return (True, "")
|
|
|
|
|
|
def _slug_from_url(url: str) -> str | None:
|
|
"""Cheap slug fallback when the scraper doesn't expose one."""
|
|
try:
|
|
parts = [p for p in urlparse(url).path.split("/") if p]
|
|
return parts[-1][:255] if parts else None
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _safe_call(fn, default=None):
|
|
"""recipe_scrapers raises various Exception subclasses for missing
|
|
fields. Swallow them per-field rather than aborting the whole scrape."""
|
|
try:
|
|
return fn()
|
|
except Exception:
|
|
return default
|
|
|
|
|
|
def _to_mealie_shape(scraper, source_url: str) -> dict:
|
|
"""Reshape a recipe_scrapers.AbstractScraper into the dict shape
|
|
forge.enrich_recipe expects (a Mealie recipe). Falls back gracefully
|
|
when individual fields are unavailable."""
|
|
title = _safe_call(scraper.title) or ""
|
|
description = _safe_call(getattr(scraper, "description", lambda: ""), "") or ""
|
|
yields = _safe_call(scraper.yields, "") or ""
|
|
image = _safe_call(scraper.image, "") or ""
|
|
|
|
ings_raw = _safe_call(scraper.ingredients, []) or []
|
|
ingredients = [
|
|
{"note": str(x).strip()}
|
|
for x in ings_raw
|
|
if x and str(x).strip()
|
|
]
|
|
|
|
# Prefer instructions_list when supported; some scrapers only expose
|
|
# the joined string.
|
|
steps_list: list[str] = []
|
|
instructions_list = _safe_call(getattr(scraper, "instructions_list", lambda: None), None)
|
|
if instructions_list:
|
|
steps_list = [str(s).strip() for s in instructions_list if s and str(s).strip()]
|
|
else:
|
|
joined = _safe_call(scraper.instructions, "") or ""
|
|
steps_list = [s.strip() for s in joined.split("\n") if s.strip()]
|
|
instructions = [{"text": s} for s in steps_list]
|
|
|
|
return {
|
|
"name": title,
|
|
"description": description,
|
|
"recipeYield": yields,
|
|
"image": image,
|
|
"source_url": source_url,
|
|
"recipeIngredient": ingredients,
|
|
"recipeInstructions": instructions,
|
|
}
|
|
|
|
|
|
def _scrape_one(url: str) -> tuple[dict, str | None] | None:
|
|
"""Scrape a single URL. Returns (mealie_shape_dict, image_url) on
|
|
success. Returns None on any unrecoverable scraper error."""
|
|
# SSRF defense-in-depth: even though /api/discover/scrape-start
|
|
# validates URLs at queue time, re-check here so any future caller
|
|
# (cron, admin script, future bulk runner) can't bypass it.
|
|
ok, reason = is_public_url(url)
|
|
if not ok:
|
|
log.warning("[discover] refusing private/restricted url %s (%s)", url, reason)
|
|
return None
|
|
|
|
try:
|
|
from recipe_scrapers import scrape_me # type: ignore
|
|
except ImportError:
|
|
log.exception("[discover] recipe_scrapers not installed")
|
|
return None
|
|
|
|
try:
|
|
scraper = scrape_me(url)
|
|
except Exception as e:
|
|
# scraper_exists_for is False or the request failed.
|
|
# Fall back to scrape_html with supported_only=False so unknown
|
|
# sites still get a JSON-LD/microdata pass.
|
|
try:
|
|
from recipe_scrapers import scrape_html # type: ignore
|
|
import requests as _rq
|
|
resp = _rq.get(
|
|
url,
|
|
timeout=15,
|
|
# allow_redirects=False: is_public_url validated the
|
|
# original host as public; a 30x to 127.0.0.1 / 169.254.x
|
|
# would otherwise route this scrape worker at internal
|
|
# services (LAN scanner, cloud metadata IMDS). Treat 30x
|
|
# as scrape failure rather than chase the redirect chain.
|
|
# The recipe_scrapers primary path has its own internal
|
|
# request chain that's a known residual — the docstring
|
|
# on is_public_url notes the long-term answer is a
|
|
# custom requests transport that re-validates per hop.
|
|
allow_redirects=False,
|
|
headers={
|
|
# Realistic desktop UA — many recipe sites 403 anything
|
|
# that smells like a bot. We're identifying as a normal
|
|
# browser; per-site robots.txt we still respect via
|
|
# recipe_scrapers' built-in wild_mode safety nets.
|
|
"User-Agent": (
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
"Chrome/120.0 Safari/537.36"
|
|
),
|
|
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
|
"Accept-Language": "en-US,en;q=0.5",
|
|
},
|
|
)
|
|
if resp.status_code != 200:
|
|
log.warning("[discover] fetch %s -> %s", url, resp.status_code)
|
|
return None
|
|
scraper = scrape_html(
|
|
html=resp.text, org_url=url, supported_only=False
|
|
)
|
|
except Exception as e2:
|
|
log.warning("[discover] scrape_me(%s) failed: %s / fallback: %s", url, e, e2)
|
|
return None
|
|
|
|
shaped = _to_mealie_shape(scraper, url)
|
|
image = shaped.get("image") or None
|
|
if not shaped.get("name"):
|
|
log.warning("[discover] no name extracted from %s", url)
|
|
return None
|
|
if not shaped.get("recipeIngredient"):
|
|
log.warning("[discover] no ingredients extracted from %s", url)
|
|
return None
|
|
return shaped, image
|
|
|
|
|
|
def run_discover(
|
|
*,
|
|
db: DB,
|
|
job_id: int,
|
|
forge: Forge,
|
|
urls: list[str],
|
|
) -> None:
|
|
"""Walk a list of URLs: scrape → insert → enrich. Runs in a daemon
|
|
thread; respects external cancel via state poll."""
|
|
log.info("[discover:%s] start (%d urls)", job_id, len(urls))
|
|
|
|
def _cancelled() -> bool:
|
|
s = db.get_discover_job_state(job_id)
|
|
return s in ("cancelled", "failed", "done")
|
|
|
|
try:
|
|
for url in urls:
|
|
if _cancelled():
|
|
log.info("[discover:%s] aborted (state changed)", job_id)
|
|
return
|
|
|
|
db.update_discover_job_progress(job_id, pages_delta=1)
|
|
|
|
scraped = _scrape_one(url)
|
|
if scraped is None:
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"scrape failed: {url[:200]}"
|
|
)
|
|
continue
|
|
shaped, image = scraped
|
|
|
|
try:
|
|
slug = _slug_from_url(url)
|
|
discover_id = db.insert_discovered_recipe(
|
|
slug=slug,
|
|
source_url=url,
|
|
name=shaped.get("name") or None,
|
|
description=(shaped.get("description") or "")[:60000] or None,
|
|
image_url=image,
|
|
scraped_json=json.dumps(shaped, ensure_ascii=False),
|
|
)
|
|
except Exception as e:
|
|
log.warning("[discover:%s] insert(%s) failed: %s", job_id, url, e)
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"insert: {str(e)[:200]}"
|
|
)
|
|
continue
|
|
|
|
if not discover_id:
|
|
# UNIQUE conflict — already in the corpus from a prior scrape
|
|
db.update_discover_job_progress(job_id, skipped_delta=1)
|
|
continue
|
|
|
|
try:
|
|
meta = forge.enrich_recipe(shaped)
|
|
except (ForgeError, RuntimeError) as e:
|
|
msg = str(e)[:500]
|
|
log.warning("[discover:%s] enrich(%s): %s", job_id, url, msg)
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"enrich: {msg[:200]}"
|
|
)
|
|
# Leave the row in 'raw' so we can retry enrichment later.
|
|
# The recipe IS in the corpus; just hasn't been classified.
|
|
continue
|
|
|
|
try:
|
|
db.update_discovered_meta(
|
|
discover_id,
|
|
meta_json=json.dumps(meta, ensure_ascii=False),
|
|
version=DB.ENRICH_VERSION,
|
|
)
|
|
db.update_discover_job_progress(job_id, added_delta=1)
|
|
except Exception as e:
|
|
log.warning("[discover:%s] persist meta(%s): %s", job_id, url, e)
|
|
db.update_discover_job_progress(
|
|
job_id, error_delta=1, last_error=f"persist: {str(e)[:200]}"
|
|
)
|
|
|
|
db.finalize_discover_job(job_id, state="done")
|
|
log.info("[discover:%s] done", job_id)
|
|
except Exception:
|
|
log.exception("[discover:%s] crashed", job_id)
|
|
try:
|
|
db.finalize_discover_job(job_id, state="failed")
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def spawn_thread(
|
|
*,
|
|
db: DB,
|
|
job_id: int,
|
|
forge: Forge,
|
|
urls: list[str],
|
|
) -> threading.Thread:
|
|
t = threading.Thread(
|
|
target=run_discover,
|
|
kwargs={"db": db, "job_id": job_id, "forge": forge, "urls": urls},
|
|
name=f"discover-recipes-{job_id}",
|
|
daemon=True,
|
|
)
|
|
t.start()
|
|
return t
|