v0.3 step 5: lean shopping list — claude on-demand foods + game strip
Two changes:
1. foods catalog grows organically. Switch the canonical seed from the
noisy USDA dump (2462 rows of "'s, classic chicken noodle soup")
to the Sonnet-curated cut (229 clean rows). search_food() is now
exact + case-insensitive — Mealie's parser already canonicalizes
food names household-side, so cauldron just needs to look them up
verbatim. On miss, the /list view calls forge.fetch_food_info() to
ask Sonnet for {density_g_per_ml, default_unit_class, common_size_g,
category}, persists the row with source='claude', and the household's
actual kitchen catalog builds itself out as Abby uses it.
Killer case verified end-to-end: "2 cups + 50g + 1.25 lb rice"
collapses to a single "2.25 lb rice" line on the shopping list once
rice has a density row.
2. Game system stripped from /plan. Scoreboard panel, streak banner,
"first to lock takes the week" / "🏆 you locked this one in" copy
all gone. award_pick_points calls in /api/plan/generate +
/api/plan/regenerate stopped firing. household_scoreboard /
household_streak DB methods kept as dead code; cauldron_pick_points
table left in place — non-destructive, easy to revive later if
gamification comes back. Goal: get the base flow (pick → plan →
list) working for Abby first, layer features on after.
This commit is contained in:
parent
36aba73f66
commit
d649b99aef
6 changed files with 2444 additions and 100 deletions
2063
cauldron/data/foods_seed.json
Normal file
2063
cauldron/data/foods_seed.json
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -1,20 +1,22 @@
|
|||
"""Foods catalog — canonical food rows + the seed loader.
|
||||
|
||||
Phase A step 1 (v0.3): seed cauldron_foods from USDA SR Legacy via the
|
||||
JSON file at cauldron/data/foods_seed_usda.json. Idempotent — running
|
||||
multiple times is fine, INSERT IGNORE on the unique canonical_name key.
|
||||
Seed = cauldron/data/foods_seed.json (Sonnet-curated, ~229 clean rows
|
||||
with proper densities and unit classes). The raw USDA dump still lives
|
||||
at foods_seed_usda.json as a reference; we don't load it directly.
|
||||
|
||||
Phase A step 2 (next commit): aggregator engine reads these rows + the
|
||||
per-household cauldron_food_mapping to group recipe ingredients.
|
||||
|
||||
Phase A step 3 (later): claude-curated cleanup of the USDA seed (better
|
||||
names, missing common foods, count-based foods like 'egg' / 'onion').
|
||||
Lookup is exact + case-insensitive. Mealie's parser already
|
||||
canonicalizes food names household-side via its own alias system, so
|
||||
the food.name we get from Mealie is consistent across recipes. When a
|
||||
Mealie food name has no match in cauldron_foods, server.py's
|
||||
ensure_food() calls clawdforge to fetch density+unit_class+common_size_g
|
||||
for that exact name, persists it with source='claude', and the
|
||||
household's catalog grows organically.
|
||||
"""
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
SEED_PATH = Path(__file__).parent / "data" / "foods_seed_usda.json"
|
||||
SEED_PATH = Path(__file__).parent / "data" / "foods_seed.json"
|
||||
|
||||
|
||||
def seed_count(db) -> int:
|
||||
|
|
@ -53,13 +55,15 @@ def _load_seed_file(db, path: Path) -> int:
|
|||
"""
|
||||
INSERT IGNORE INTO cauldron_foods
|
||||
(canonical_name, category, density_g_per_ml,
|
||||
default_unit_class, usda_fdc_id, usda_description, source)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, 'usda')
|
||||
common_size_g, default_unit_class, usda_fdc_id,
|
||||
usda_description, source)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, 'usda')
|
||||
""",
|
||||
(
|
||||
entry["canonical_name"][:255],
|
||||
entry.get("category"),
|
||||
entry.get("density_g_per_ml"),
|
||||
entry.get("common_size_g"),
|
||||
entry.get("default_unit_class") or "mass",
|
||||
entry.get("usda_fdc_id"),
|
||||
(entry.get("usda_description") or "")[:500],
|
||||
|
|
@ -72,25 +76,65 @@ def _load_seed_file(db, path: Path) -> int:
|
|||
return inserted
|
||||
|
||||
|
||||
def search_food(db, name: str, *, limit: int = 5) -> list[dict]:
|
||||
"""Best-effort canonical food lookup by name (used by aggregator + UI)."""
|
||||
def search_food(db, name: str, *, limit: int = 1) -> list[dict]:
|
||||
"""Case-insensitive exact match. Returns [] on miss.
|
||||
|
||||
`limit` kept for backwards-compat with callers; only ever returns 0 or 1.
|
||||
"""
|
||||
name_clean = (name or "").strip().lower()
|
||||
if not name_clean:
|
||||
return []
|
||||
with db.conn() as c, c.cursor() as cur:
|
||||
cur.execute(
|
||||
"""SELECT id, canonical_name, category, density_g_per_ml,
|
||||
default_unit_class, common_size_g, source
|
||||
FROM cauldron_foods
|
||||
WHERE LOWER(canonical_name) = %s
|
||||
LIMIT 1""",
|
||||
(name_clean,),
|
||||
)
|
||||
row = cur.fetchone()
|
||||
return [dict(row)] if row else []
|
||||
|
||||
|
||||
def upsert_claude_food(
|
||||
db,
|
||||
*,
|
||||
canonical_name: str,
|
||||
density_g_per_ml: float | None,
|
||||
default_unit_class: str,
|
||||
common_size_g: float | None,
|
||||
category: str | None = None,
|
||||
) -> dict:
|
||||
"""Insert (or update if a row already exists) a canonical food row from
|
||||
a clawdforge response. Returns the row as a dict. Idempotent."""
|
||||
name = canonical_name.strip().lower()[:255]
|
||||
cls = (default_unit_class or "mass").lower()
|
||||
if cls not in ("mass", "volume", "count", "mixed"):
|
||||
cls = "mass"
|
||||
with db.conn() as c, c.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id, canonical_name, category, density_g_per_ml,
|
||||
default_unit_class, common_size_g
|
||||
FROM cauldron_foods
|
||||
WHERE canonical_name LIKE %s OR canonical_name LIKE %s
|
||||
ORDER BY
|
||||
(CASE WHEN canonical_name = %s THEN 0
|
||||
WHEN canonical_name LIKE %s THEN 1
|
||||
ELSE 2 END),
|
||||
CHAR_LENGTH(canonical_name)
|
||||
LIMIT %s
|
||||
INSERT INTO cauldron_foods
|
||||
(canonical_name, category, density_g_per_ml,
|
||||
common_size_g, default_unit_class, source)
|
||||
VALUES (%s, %s, %s, %s, %s, 'claude')
|
||||
ON DUPLICATE KEY UPDATE
|
||||
category=COALESCE(VALUES(category), category),
|
||||
density_g_per_ml=COALESCE(VALUES(density_g_per_ml), density_g_per_ml),
|
||||
common_size_g=COALESCE(VALUES(common_size_g), common_size_g),
|
||||
default_unit_class=VALUES(default_unit_class),
|
||||
source='claude'
|
||||
""",
|
||||
(f"{name}%", f"%{name}%", name, f"{name}%", limit),
|
||||
(name, category, density_g_per_ml, common_size_g, cls),
|
||||
)
|
||||
return [dict(r) for r in cur.fetchall()]
|
||||
cur.execute(
|
||||
"""SELECT id, canonical_name, category, density_g_per_ml,
|
||||
default_unit_class, common_size_g, source
|
||||
FROM cauldron_foods WHERE LOWER(canonical_name)=%s LIMIT 1""",
|
||||
(name,),
|
||||
)
|
||||
return dict(cur.fetchone())
|
||||
|
||||
|
||||
def get_food(db, food_id: int) -> dict | None:
|
||||
|
|
|
|||
|
|
@ -215,6 +215,73 @@ class Forge:
|
|||
)
|
||||
|
||||
|
||||
def fetch_food_info(self, name: str, *, model: str | None = None) -> dict:
|
||||
"""Ask Sonnet for density + unit class + common size of a single
|
||||
food. Returns a dict shaped like:
|
||||
|
||||
{"density_g_per_ml": 1.04 | null,
|
||||
"default_unit_class": "mass"|"volume"|"count",
|
||||
"common_size_g": 150.0 | null,
|
||||
"category": "produce"|"dairy"|... | null}
|
||||
|
||||
density_g_per_ml is null when the food doesn't sensibly convert
|
||||
between mass and volume (e.g., whole onions, eggs — these are
|
||||
count-style). common_size_g lets the aggregator handle "1 onion"
|
||||
as a count → mass conversion. Cheap call, cached forever once
|
||||
persisted to cauldron_foods.
|
||||
"""
|
||||
prompt = (
|
||||
f"Give nutritional/cooking metadata for the food: {name!r}.\n\n"
|
||||
"Output JSON ONLY, no prose: "
|
||||
'{"density_g_per_ml": float|null, '
|
||||
'"default_unit_class": "mass"|"volume"|"count", '
|
||||
'"common_size_g": float|null, '
|
||||
'"category": "produce"|"dairy"|"meat"|"grain"|"baking"|"pantry"'
|
||||
'|"spice"|"oil"|"beverage"|"other"|null}\n\n'
|
||||
"Rules:\n"
|
||||
"- density_g_per_ml: typical packed/cooking density. Null if "
|
||||
"the food is count-based (whole onions, eggs).\n"
|
||||
"- default_unit_class: how this food is most often measured "
|
||||
"(salt=mass; milk=volume; egg=count).\n"
|
||||
"- common_size_g: the typical mass of one whole unit (1 onion "
|
||||
"≈ 150g; 1 egg ≈ 50g). Null if the food isn't naturally counted.\n"
|
||||
"- category: best single fit; null if uncertain.\n"
|
||||
)
|
||||
result = self.run(prompt, model=model or "sonnet", timeout_secs=60)
|
||||
return _extract_food_info(result)
|
||||
|
||||
|
||||
def _extract_food_info(forge_result: dict) -> dict:
|
||||
"""Normalize clawdforge wrapper → food info dict. Defensive on shapes."""
|
||||
if not isinstance(forge_result, dict):
|
||||
raise ForgeError("forge result not a dict")
|
||||
inner = forge_result.get("result", forge_result)
|
||||
if isinstance(inner, str):
|
||||
inner = _parse_json_blob(inner)
|
||||
if not isinstance(inner, dict):
|
||||
raise ForgeError(f"forge result not a dict: {str(inner)[:200]}")
|
||||
|
||||
cls = (inner.get("default_unit_class") or "mass").strip().lower()
|
||||
if cls not in ("mass", "volume", "count", "mixed"):
|
||||
cls = "mass"
|
||||
|
||||
def _f(v):
|
||||
if v is None:
|
||||
return None
|
||||
try:
|
||||
x = float(v)
|
||||
return x if x > 0 else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
return {
|
||||
"density_g_per_ml": _f(inner.get("density_g_per_ml")),
|
||||
"default_unit_class": cls,
|
||||
"common_size_g": _f(inner.get("common_size_g")),
|
||||
"category": (inner.get("category") or None) and str(inner["category"])[:64],
|
||||
}
|
||||
|
||||
|
||||
def _extract_plan_slots(forge_result: dict):
|
||||
"""clawdforge wraps its return; the JSON we asked for can sit in a few
|
||||
different shapes. Normalize aggressively."""
|
||||
|
|
|
|||
|
|
@ -501,8 +501,6 @@ def create_app() -> Flask:
|
|||
|
||||
plan = db.get_or_create_plan(hid, this_monday)
|
||||
db.enrich_plan_with_slots(plan)
|
||||
scoreboard = db.household_scoreboard(hid)
|
||||
streak = db.household_streak(hid)
|
||||
pick_count = len(db.list_household_pick_slugs(hid))
|
||||
|
||||
# Resolve display names for any subs we render — locked_by + every
|
||||
|
|
@ -523,8 +521,6 @@ def create_app() -> Flask:
|
|||
locked_by_display=locked_by_display,
|
||||
generated_by_display=generated_by_display,
|
||||
sub_display=sub_display,
|
||||
scoreboard=scoreboard,
|
||||
streak=streak,
|
||||
current_user_sub=u["sub"],
|
||||
pick_count=pick_count,
|
||||
active="plan",
|
||||
|
|
@ -611,15 +607,6 @@ def create_app() -> Flask:
|
|||
"plan": _plan_payload(plan),
|
||||
}), 409
|
||||
|
||||
# Award 1pt per pick that landed (one row per (sub, pick_used) per
|
||||
# plan, deduped client-side: a pick with two pickers gives both 1pt).
|
||||
for s in slots:
|
||||
for sub in s.get("picker_subs") or []:
|
||||
try:
|
||||
db.award_pick_points(hid, plan["id"], sub, 1, "pick_used")
|
||||
except Exception as exc:
|
||||
app.logger.warning("award_pick_points failed for %s: %s", sub, exc)
|
||||
|
||||
plan = db.mark_plan_generated(plan["id"], u["sub"])
|
||||
db.enrich_plan_with_slots(plan)
|
||||
return jsonify({"ok": True, "plan": _plan_payload(plan)})
|
||||
|
|
@ -678,12 +665,6 @@ def create_app() -> Flask:
|
|||
return jsonify({"error": "forge_failed", "detail": str(e)}), 502
|
||||
|
||||
db.save_plan_slots(plan["id"], slots)
|
||||
for s in slots:
|
||||
for sub in s.get("picker_subs") or []:
|
||||
try:
|
||||
db.award_pick_points(hid, plan["id"], sub, 1, "pick_used")
|
||||
except Exception as exc:
|
||||
app.logger.warning("award_pick_points failed for %s: %s", sub, exc)
|
||||
plan = db.mark_plan_generated(plan["id"], u["sub"])
|
||||
db.enrich_plan_with_slots(plan)
|
||||
return jsonify({"ok": True, "plan": _plan_payload(plan)})
|
||||
|
|
@ -739,9 +720,39 @@ def create_app() -> Flask:
|
|||
original_text=ri.get("display") or _ing_render(qty, unit, food_name, note),
|
||||
))
|
||||
|
||||
# foods_lookup: exact match → clawdforge fallback that persists.
|
||||
# Per-request cache so the same food in multiple recipes isn't
|
||||
# re-queried within one /list render.
|
||||
lookup_cache: dict[str, dict | None] = {}
|
||||
|
||||
def foods_lookup(name: str):
|
||||
hits = foods.search_food(db, name, limit=1)
|
||||
return hits[0] if hits else None
|
||||
key = (name or "").strip().lower()
|
||||
if not key:
|
||||
return None
|
||||
if key in lookup_cache:
|
||||
return lookup_cache[key]
|
||||
hits = foods.search_food(db, key, limit=1)
|
||||
if hits:
|
||||
lookup_cache[key] = hits[0]
|
||||
return hits[0]
|
||||
# Miss — ask clawdforge, persist, return. On any failure, cache
|
||||
# None for this request so we don't spam the model.
|
||||
try:
|
||||
info = forge.fetch_food_info(key)
|
||||
row = foods.upsert_claude_food(
|
||||
db,
|
||||
canonical_name=key,
|
||||
density_g_per_ml=info.get("density_g_per_ml"),
|
||||
default_unit_class=info.get("default_unit_class") or "mass",
|
||||
common_size_g=info.get("common_size_g"),
|
||||
category=info.get("category"),
|
||||
)
|
||||
lookup_cache[key] = row
|
||||
return row
|
||||
except Exception as exc:
|
||||
app.logger.warning("forge.fetch_food_info(%r) failed: %s", key, exc)
|
||||
lookup_cache[key] = None
|
||||
return None
|
||||
|
||||
lines = aggregator.aggregate(raw_ings, foods_lookup)
|
||||
return render_template(
|
||||
|
|
|
|||
|
|
@ -93,9 +93,9 @@
|
|||
· {{ plan.locked_at.strftime('%a %-I:%M %p') }}
|
||||
{% elif plan.slots %}
|
||||
generated{% if generated_by_display %} by <span style="color: var(--green-bright);">{{ generated_by_display }}</span>{% endif %}.
|
||||
lock when ready — first to lock takes the week.
|
||||
lock it in when the household has agreed.
|
||||
{% else %}
|
||||
{{ pick_count }} pinned in the pool. summon the planner to build a 7-day plan, then race to lock.
|
||||
{{ pick_count }} pinned in the pool. summon the planner to build a 7-day plan.
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -115,15 +115,12 @@
|
|||
|
||||
{% if plan.locked_at %}
|
||||
<p>this week's plan is locked. it'll archive Sunday night and a fresh week opens Monday.</p>
|
||||
{% if plan.locked_reason == 'user' and plan.locked_by_sub == current_user_sub %}
|
||||
<p class="muted">🏆 you locked this one in.</p>
|
||||
{% endif %}
|
||||
{% elif not plan.slots %}
|
||||
<p>no plan yet. summon claude to build one from the {{ pick_count }} pinned pick{{ '' if pick_count == 1 else 's' }} + the rest of the grimoire.</p>
|
||||
<button class="gen-cta" type="button" id="gen-btn" onclick="generatePlan(this)">🪄 generate this week's plan</button>
|
||||
<div class="gen-meta" id="gen-meta">sonnet · ~30s</div>
|
||||
{% else %}
|
||||
<p>plan's set. lock it in to claim the week — first to lock wins.</p>
|
||||
<p>plan's set.</p>
|
||||
<div class="btn-row">
|
||||
<button class="btn btn-purple" type="button" onclick="lockPlan(this)">🔒 lock this week</button>
|
||||
{% if plan.generated_by_sub == current_user_sub %}
|
||||
|
|
@ -159,53 +156,6 @@
|
|||
</section>
|
||||
{% endif %}
|
||||
|
||||
<section class="panel">
|
||||
<div class="panel-head">
|
||||
<h2>scoreboard</h2>
|
||||
<span class="ctx">picks landed · weeks won</span>
|
||||
</div>
|
||||
|
||||
{% if streak and streak.count >= 2 %}
|
||||
<p style="font-size: 1.05em; margin: .2em 0 1em 0;">
|
||||
🔥 <strong style="color: var(--green-bright);">{{ streak.display_name }}</strong>
|
||||
on a <strong>{{ streak.count }}-week</strong> run.
|
||||
</p>
|
||||
{% endif %}
|
||||
|
||||
{% if scoreboard and scoreboard|selectattr("points")|list or scoreboard and scoreboard|selectattr("wins")|list %}
|
||||
<table style="width: 100%; border-collapse: collapse; font-size: .95em;">
|
||||
<thead>
|
||||
<tr style="border-bottom: 1px solid var(--line);">
|
||||
<th style="text-align: left; color: var(--purple); font-size: 11px; letter-spacing: .15em; text-transform: uppercase; font-family: var(--mono); font-weight: 600; padding: 6px 0;">member</th>
|
||||
<th style="text-align: right; color: var(--purple); font-size: 11px; letter-spacing: .15em; text-transform: uppercase; font-family: var(--mono); font-weight: 600; padding: 6px 0;">pts</th>
|
||||
<th style="text-align: right; color: var(--purple); font-size: 11px; letter-spacing: .15em; text-transform: uppercase; font-family: var(--mono); font-weight: 600; padding: 6px 0;">wins</th>
|
||||
<th style="text-align: right; color: var(--purple); font-size: 11px; letter-spacing: .15em; text-transform: uppercase; font-family: var(--mono); font-weight: 600; padding: 6px 0;">last</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for s in scoreboard %}
|
||||
<tr style="border-bottom: 1px solid var(--line-soft);">
|
||||
<td style="padding: 8px 0;">
|
||||
{{ loop.index }}.
|
||||
<strong style="color: {% if loop.first and (s.points or s.wins) %}var(--green-bright){% else %}var(--bone){% endif %};">
|
||||
{{ s.display_name or s.email.split('@')[0] }}
|
||||
</strong>
|
||||
{% if s.sub == current_user_sub %}<span class="muted">· you</span>{% endif %}
|
||||
</td>
|
||||
<td style="text-align: right; padding: 8px 0; font-family: var(--mono); color: var(--bone);">{{ s.points or 0 }}</td>
|
||||
<td style="text-align: right; padding: 8px 0; font-family: var(--mono); color: var(--bone);">{{ s.wins or 0 }}</td>
|
||||
<td style="text-align: right; padding: 8px 0; font-family: var(--mono); color: var(--muted); font-size: .85em;">
|
||||
{{ s.last_win.strftime('%b %-d') if s.last_win else '—' }}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
{% else %}
|
||||
<p class="muted">nobody's locked or generated yet. be the first.</p>
|
||||
{% endif %}
|
||||
</section>
|
||||
|
||||
<script>
|
||||
async function lockPlan(btn) {
|
||||
btn.disabled = true; btn.textContent = '…';
|
||||
|
|
@ -243,7 +193,7 @@ async function generatePlan(btn) {
|
|||
}
|
||||
|
||||
async function rerollPlan(btn) {
|
||||
if (!confirm('re-roll the week? slots + points reset.')) return;
|
||||
if (!confirm('re-roll the week? slots will be replaced.')) return;
|
||||
btn.disabled = true; btn.textContent = '…';
|
||||
try {
|
||||
const r = await fetch('/api/plan/regenerate', { method: 'POST' });
|
||||
|
|
|
|||
209
scripts/clean_foods_seed.py
Normal file
209
scripts/clean_foods_seed.py
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Clean the USDA-derived foods seed via clawdforge → Sonnet, in batches.
|
||||
|
||||
Input: cauldron/data/foods_seed_usda.json (~2462 noisy rows)
|
||||
Output: cauldron/data/foods_seed.json (curated, ~500-800 rows)
|
||||
|
||||
Why batched: 2462 entries × 200 chars = 577KB prompt; Sonnet hit timeout
|
||||
on the single-shot curation. Splitting by category keeps each batch ~30-80
|
||||
entries → ~50KB prompt → ~30-60s round-trip.
|
||||
|
||||
Run with:
|
||||
CLAWDFORGE_TOKEN=cf_... python3 scripts/clean_foods_seed.py
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
HERE = Path(__file__).parent.parent
|
||||
RAW_PATH = HERE / "cauldron/data/foods_seed_usda.json"
|
||||
OUT_PATH = HERE / "cauldron/data/foods_seed.json"
|
||||
|
||||
CLAWDFORGE_URL = os.environ.get("CLAWDFORGE_URL", "http://192.168.0.5:8800")
|
||||
CLAWDFORGE_TOKEN = os.environ["CLAWDFORGE_TOKEN"]
|
||||
|
||||
|
||||
# Suggested count-foods to add per category — Sonnet uses these as seeds
|
||||
# but is free to add more.
|
||||
COUNT_FOODS_HINTS = {
|
||||
"produce-vegetable": "onion (~150g), garlic clove (~5g), tomato (~120g), "
|
||||
"potato (~170g), bell pepper (~120g), jalapeno (~14g), "
|
||||
"shallot (~25g), carrot (~60g), leek (~90g)",
|
||||
"produce-fruit": "apple (~180g), banana (~118g), orange (~130g), "
|
||||
"lemon (~60g), lime (~65g), avocado (~200g), peach (~150g)",
|
||||
"dairy": "egg (~50g large, count not mass), slice cheese (~28g)",
|
||||
"grain": "slice bread (~28g), tortilla (~50g)",
|
||||
"meat": "(no count hints — meat is sold by weight)",
|
||||
"legume": "can (~425g drained tomato/bean/etc when relevant)",
|
||||
"condiment": "can (~400g for canned tomato/coconut milk)",
|
||||
"oil-fat": "(none — sold by volume or weight)",
|
||||
"spice": "(none — pinch/dash for to-taste)",
|
||||
"baking": "(none unless slice-of-X applies)",
|
||||
"beverage": "(none — bought in bottles, treat as volume)",
|
||||
"nut-seed": "(none — sold by weight)",
|
||||
"other": "(skip count hints)",
|
||||
}
|
||||
|
||||
|
||||
SYSTEM_PROMPT = """You are a culinary database curator. You receive a small batch
|
||||
of raw USDA-derived food entries (one category at a time) and produce a clean,
|
||||
useful subset for a family meal-planning app.
|
||||
|
||||
OUTPUT: ONE valid JSON object. No markdown fences, no prose, JSON only:
|
||||
|
||||
{
|
||||
"foods": [
|
||||
{
|
||||
"canonical_name": "<short singular noun, lowercase>",
|
||||
"category": "<input_category>",
|
||||
"density_g_per_ml": <number or null>,
|
||||
"default_unit_class": "<one of: mass, volume, count, mixed>",
|
||||
"common_size_g": <number or null>,
|
||||
"usda_fdc_id": <int or null>,
|
||||
"notes": <string or null>
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
RULES:
|
||||
|
||||
1. **Drop ruthlessly** — composite/prepared meals, brand-laden entries
|
||||
(PILLSBURY ..., GERBER ...), babyfood, alcoholic beverages, fast food,
|
||||
ready-to-eat junk, sulfured/preserved derivatives.
|
||||
|
||||
2. **Normalize canonical_name** to the simplest cooking form:
|
||||
- "Apples, raw" → "apple"
|
||||
- "Rice, white, long-grain, regular, raw" → "white rice" (or "rice")
|
||||
- "Pepper, black, ground" → "black pepper"
|
||||
- "Mayonnaise, reduced fat, with olive oil" → DROP (variant)
|
||||
- Keep meaningful distinctions ("brown rice" vs "white rice", "salted butter" vs "unsalted butter")
|
||||
|
||||
3. **Preserve density_g_per_ml** from the input — don't re-derive.
|
||||
|
||||
4. **default_unit_class**:
|
||||
- mass: dry goods sold by weight (rice, flour, sugar, meat, beans, butter)
|
||||
- volume: liquids (milk, oil, juice, syrup, vinegar)
|
||||
- count: discrete items (egg, onion, garlic clove, lemon, slice bread)
|
||||
- mixed: bought in different forms (cheese — block vs shredded; salt — pinch vs grams)
|
||||
|
||||
5. **For count foods, set common_size_g** so the aggregator can convert
|
||||
"2 onions + 1 cup chopped onion" sensibly.
|
||||
|
||||
6. **ADD common count-based foods USDA doesn't track for this category**
|
||||
if they're missing. Suggested hints will be supplied per category.
|
||||
|
||||
7. Cap output at **80 foods per category**. Quality over quantity. Drop
|
||||
variants — pick the canonical form and skip the rest.
|
||||
|
||||
8. JSON only. No markdown fences. No preamble."""
|
||||
|
||||
|
||||
USER_PROMPT_TEMPLATE = """Curate the **{category}** entries below.
|
||||
|
||||
Input: {n} raw entries.
|
||||
|
||||
Suggested count-foods to add for this category if missing:
|
||||
{hints}
|
||||
|
||||
Entries:
|
||||
{json}"""
|
||||
|
||||
|
||||
def main():
|
||||
raw = json.loads(RAW_PATH.read_text())
|
||||
print(f"Loaded {len(raw)} raw foods", file=sys.stderr)
|
||||
|
||||
# Bucket by category
|
||||
by_cat: dict[str, list[dict]] = defaultdict(list)
|
||||
for r in raw:
|
||||
by_cat[r.get("category") or "other"].append(r)
|
||||
|
||||
print(f"Categories: {[(c, len(items)) for c, items in sorted(by_cat.items(), key=lambda x: -len(x[1]))]}", file=sys.stderr)
|
||||
print(file=sys.stderr)
|
||||
|
||||
all_foods: list[dict] = []
|
||||
seen_canonical: set[str] = set()
|
||||
total_dropped = 0
|
||||
|
||||
for cat in sorted(by_cat.keys()):
|
||||
items = by_cat[cat]
|
||||
# 'other' is too noisy + low-priority — process last and let Sonnet drop ~all
|
||||
# we'll run it but cap at first 100 entries to keep prompt size sane
|
||||
slice_items = items[:120] if cat != "other" else items[:80]
|
||||
|
||||
prompt = USER_PROMPT_TEMPLATE.format(
|
||||
category=cat,
|
||||
n=len(slice_items),
|
||||
hints=COUNT_FOODS_HINTS.get(cat, "(none)"),
|
||||
json=json.dumps(slice_items, ensure_ascii=False),
|
||||
)
|
||||
|
||||
print(f"[{cat}] {len(slice_items)} entries → ", end="", file=sys.stderr, flush=True)
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
r = requests.post(
|
||||
f"{CLAWDFORGE_URL.rstrip('/')}/run",
|
||||
headers={"Authorization": f"Bearer {CLAWDFORGE_TOKEN}"},
|
||||
json={
|
||||
"prompt": prompt,
|
||||
"system": SYSTEM_PROMPT,
|
||||
"model": "sonnet",
|
||||
"timeout_secs": 180,
|
||||
},
|
||||
timeout=210,
|
||||
)
|
||||
except requests.RequestException as e:
|
||||
print(f"transport err: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
dur = time.monotonic() - t0
|
||||
|
||||
if r.status_code >= 400:
|
||||
print(f"HTTP {r.status_code} ({dur:.1f}s) body={r.text[:200]}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
body = r.json()
|
||||
if not body.get("ok"):
|
||||
print(f"forge !ok ({dur:.1f}s) {body.get('error', '')}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
result = body.get("result")
|
||||
if isinstance(result, str):
|
||||
try:
|
||||
result = json.loads(result)
|
||||
except json.JSONDecodeError:
|
||||
print(f"non-JSON result ({dur:.1f}s)", file=sys.stderr)
|
||||
continue
|
||||
|
||||
foods = result.get("foods") or [] if isinstance(result, dict) else []
|
||||
kept = 0
|
||||
for f in foods:
|
||||
cn = (f.get("canonical_name") or "").strip().lower()
|
||||
if not cn or cn in seen_canonical:
|
||||
continue
|
||||
f["canonical_name"] = cn
|
||||
f["category"] = cat # enforce category from outer batch
|
||||
seen_canonical.add(cn)
|
||||
all_foods.append(f)
|
||||
kept += 1
|
||||
dropped = len(slice_items) - kept
|
||||
total_dropped += max(0, dropped)
|
||||
print(f"{kept} kept, ~{max(0, dropped)} dropped ({dur:.1f}s)", file=sys.stderr)
|
||||
|
||||
all_foods.sort(key=lambda x: (x["category"] or "", x["canonical_name"]))
|
||||
|
||||
print(file=sys.stderr)
|
||||
print(f"Total cleaned: {len(all_foods)} foods", file=sys.stderr)
|
||||
|
||||
OUT_PATH.write_text(json.dumps(all_foods, indent=2, ensure_ascii=False) + "\n")
|
||||
print(f"Wrote {OUT_PATH}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue