diff --git a/Dockerfile b/Dockerfile index 59937e9..7caf20b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -254,10 +254,20 @@ ENV PATH=/home/crafter/.composer/vendor/bin:$PATH COPY --chown=crafter:crafter smoke.sh /usr/local/bin/smoke.sh USER root RUN chmod +x /usr/local/bin/smoke.sh -USER crafter # ============================================================ -# 21. Final ENV / WORKDIR / CMD +# 21. Application — FastAPI + async runner (wave 1: steps 2+3+4) # ============================================================ +COPY pyproject.toml requirements.txt /app/ +RUN pip install --break-system-packages --no-cache-dir -r /app/requirements.txt +COPY --chown=crafter:crafter crafting_table /app/crafting_table +RUN chown -R crafter:crafter /app + +# ============================================================ +# 22. Final ENV / WORKDIR / CMD +# ============================================================ +USER crafter WORKDIR /workspace -CMD ["/bin/bash"] +ENV PYTHONPATH=/app \ + PYTHONUNBUFFERED=1 +CMD ["uvicorn", "crafting_table.server:app", "--host", "0.0.0.0", "--port", "8810"] diff --git a/README.md b/README.md index f78e813..34c6aaa 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,10 @@ Polyglot dev/build/audit container — the build farm for the Sulkta ecosystem. ## What this is -A single Docker container with every toolchain we work with, used as a -reliable place to compile / test / audit any Sulkta repo regardless of -where the caller is — agents, Claude sessions, ad-hoc curl, scheduled cron. +A single Docker container with every toolchain we work with, fronted by a +FastAPI HTTP API + async job runner. Used as a reliable place to compile / +test / audit any Sulkta repo regardless of where the caller is — agents, +Claude sessions, ad-hoc curl, scheduled cron. Eventual surface (v0.1 full): HTTP API + MCP server + project registry + job runner + structured findings + email digest + autonomous patch loop @@ -14,12 +15,12 @@ through clawdforge. Spec: `Sulkta-Coop/openclaw-workspace/memory/spec-crafting-table.md` (LAN-only). -## Status — v0.1 step 1 of 10 +## Status — v0.1 - [x] Step 1: Dockerfile + per-language smoke -- [ ] Step 2: SQLite ledger + project registry -- [ ] Step 3: HTTP API skeleton (FastAPI, port 8810) -- [ ] Step 4: Job runner core (asyncio worker pool) +- [x] Step 2: SQLite ledger + project registry +- [x] Step 3: HTTP API skeleton (FastAPI, port 8810) +- [x] Step 4: Job runner core (asyncio worker pool, git worktree, subprocess) - [ ] Step 5: Per-language parsers (Rust / Python / Go / TS first) - [ ] Step 6: Findings extraction + storage - [ ] Step 7: MCP server (stdio JSON-RPC, 8 tools) @@ -48,39 +49,177 @@ Spec: `Sulkta-Coop/openclaw-workspace/memory/spec-crafting-table.md` (LAN-only). | Bash | bash + shellcheck + bats + shfmt | | Generic | git, jq, yq, ripgrep, fd, gh-cli, curl, wget | +## HTTP API surface + +LAN-only. Every request needs `Authorization: Bearer `. The default +LAN allowlist is `10/8`, `172.16/12`, `192.168/16`, `127/8`, `::1/128`; +override via `CRAFTING_LAN_CIDRS`. + +| Method | Path | Who | What | +|--------|-----------------------------------|--------|--------------------------------| +| GET | `/healthz` | LAN | Liveness + runner stats | +| POST | `/admin/tokens` | admin | Mint a new bearer | +| GET | `/admin/tokens` | admin | List tokens | +| DELETE | `/admin/tokens/{name}` | admin | Revoke a token | +| POST | `/projects` | any | Register (becomes owner) | +| GET | `/projects` | any | List own (or all if admin) | +| GET | `/projects/{name}` | owner | Detail (404 if not visible) | +| PUT | `/projects/{name}` | owner | Update | +| DELETE | `/projects/{name}` | owner | Remove (cascades jobs+findings)| +| POST | `/projects/{name}/jobs` | owner | Enqueue a recipe run | +| GET | `/jobs?project=&status=&limit=` | any | List own (or all if admin) | +| GET | `/jobs/{id}` | owner | State + last 200 log lines | +| GET | `/jobs/{id}/log` | owner | Full log (file stream) | +| GET | `/jobs/{id}/findings` | owner | Structured findings (wave 1: empty) | + +Cross-token access returns **404, not 403** — same existence-leak guard as +clawdforge sessions. + +## Quickstart + +After build + first boot, the admin bearer is written to +`/data/admin-bearer.txt` (chmod 600 inside the container — readable from +the bind-mounted appdata path on the host). + +```bash +ADMIN=$(cat /mnt/user/appdata/crafting-table/data/admin-bearer.txt) + +# Mint a project-scoped token +TOKEN=$(curl -s http://192.168.0.5:8810/admin/tokens \ + -H "Authorization: Bearer $ADMIN" \ + -H "content-type: application/json" \ + -d '{"name":"clawdforge","is_admin":false,"ip_cidrs":[]}' | jq -r .bearer) + +# Register the project +curl -s http://192.168.0.5:8810/projects \ + -H "Authorization: Bearer $TOKEN" \ + -H "content-type: application/json" \ + -d '{ + "name": "clawdforge", + "git_url": "http://192.168.0.5:3001/Sulkta-Coop/clawdforge.git", + "default_branch": "main", + "languages": ["python", "rust"], + "subprojects": [ + {"path": "clients/python", "language": "python", + "test": "pytest tests/", "lint": "ruff check .", "audit": "pip-audit", + "timeout_secs": 600}, + {"path": "clients/rust", "language": "rust", + "build": "cargo build --release", "test": "cargo test --all", + "audit": "cargo audit", "timeout_secs": 1800} + ] + }' + +# Kick off a test job +JOB=$(curl -s http://192.168.0.5:8810/projects/clawdforge/jobs \ + -H "Authorization: Bearer $TOKEN" \ + -H "content-type: application/json" \ + -d '{"recipe":"test","subproject":"clients/python"}' | jq -r .job_id) + +# Poll status +watch -n2 "curl -s http://192.168.0.5:8810/jobs/$JOB \ + -H 'Authorization: Bearer $TOKEN' | jq '.job.status, .log_tail[-5:]'" + +# Stream the full log +curl http://192.168.0.5:8810/jobs/$JOB/log \ + -H "Authorization: Bearer $TOKEN" +``` + ## Build + smoke ```bash docker network inspect sulkta >/dev/null 2>&1 || docker network create sulkta docker compose build -docker compose up -# expect: "=== ALL TOOLCHAINS GREEN ===" then exit 0 -``` -The smoke compiles + runs a hello-world in every language. If it exits 0, -the image is good. +# Run the per-toolchain hello-world smoke +docker compose run --rm crafting-table /usr/local/bin/smoke.sh +# expect: "=== ALL TOOLCHAINS GREEN ===" + +# Bring up the API +docker compose up -d +curl http://192.168.0.5:8810/healthz +``` ## Image notes - Base: `debian:bookworm-slim`. Swift uses the upstream Ubuntu 22.04 tarball which links against bookworm's libicu/libstdc++ baseline. -- Runs as non-root user `crafter` (uid 1000) with passwordless sudo. -- Volume mount points: `/workspace`, `/caches/{cargo,maven,gradle,npm,pip,bun}`, - `/data`. Compose binds these to named volumes so they survive `compose down`. +- Runs as non-root user `crafter` (uid 1000) with passwordless sudo. The + API server runs as `crafter`. Recipe commands run as `crafter` too — + never elevated to root. +- Volume mount points (production): + - `/data` — SQLite ledger, admin bearer file, per-job logs + - `/workspace` — bare clones + per-job worktrees + - `/caches` — cargo / maven / gradle / npm / pip / bun caches - Network: external `sulkta` bridge (same one clawdforge + cauldron use). Create with `docker network create sulkta` if missing. - Image size baseline is large (8-15 GB expected). Per spec: that's fine. +## Architecture notes + +### Recipe security +Recipe commands run via `/bin/sh -c` so any shell metachar works. This is +**by design** — admins set them. Recipes like `cargo build && cargo test` +work as expected; `; rm -rf /` would too if an admin set it. The container +sandbox + `crafter` user are the safety net, not the recipe parser. + +### Workspace strategy +- `/workspace//.cache/` — bare clone of the upstream +- `/workspace///` — git worktree pointing at the requested + branch+sha, removed after the job ends +- Periodic gc (1h timer) prunes worktrees older than 24h and runs + `git gc --prune=now` on bare clones quiet for >7d + +### Recipe immutability +Every job snapshots the project's recipe at run-time +(`recipe_snapshot_json`). Editing a project's recipe doesn't retcon the +view of in-flight or already-finished jobs. + +### Process group + timeout +Recipe subprocesses spawn in their own process group via +`start_new_session=True`. On timeout we `os.killpg(pgid, SIGTERM)` and grace +for 10s before escalating to `SIGKILL`. Without process-group kill, +multi-process recipes (cargo build spawning rustc, etc.) could leave +orphans that hold the stdout pipe open. + +### Restart resilience +Jobs marked `running` but stranded by a process crash are NOT auto-resumed. +On startup the runner sweeps them to `failed` with `exit_code=-1` and +appends a synthetic log line `[crafting-table] runner restart, job +orphaned`. Callers can re-enqueue if they want. + +### SQLite + WAL +Single host, single process — SQLite is plenty. WAL mode (`PRAGMA +journal_mode=WAL` + `synchronous=NORMAL`) gives many readers + one writer +without lock contention. The runner is the only mutator of `jobs`/ +`findings`; HTTP workers mostly read. + ## Layout ``` . -├── Dockerfile # monolith image with all toolchains -├── compose.yml # build + run-smoke wiring -├── smoke.sh # per-language hello-world test, baked in at /usr/local/bin/smoke.sh -├── README.md -├── LICENSE # MIT -└── .gitignore +├── Dockerfile # monolith image: toolchains + Python app +├── compose.yml # build + run wiring +├── smoke.sh # per-language hello-world test +├── crafting_table/ +│ ├── server.py # FastAPI app + endpoints +│ ├── db.py # SQLite + migrations +│ ├── auth.py # bearer + IP allowlist +│ ├── runner.py # async job pool + subprocess exec +│ ├── workspace.py # bare clone + worktree materialization + gc +│ ├── models.py # Pydantic schemas +│ └── config.py # env-driven config +├── tests/ # pytest suite (~60 tests) +├── pyproject.toml +├── requirements.txt +└── .env.example +``` + +## Tests + +```bash +python3 -m venv .venv && source .venv/bin/activate +pip install -e '.[test]' +pytest tests/ ``` ## License diff --git a/compose.yml b/compose.yml index 53bea86..ce57b43 100644 --- a/compose.yml +++ b/compose.yml @@ -1,7 +1,12 @@ -# crafting-table v0.1 — step 1 compose. +# crafting-table v0.1 — wave 1 compose (steps 2+3+4 wired in). # -# Builds the monolith image and runs the smoke test once. -# In step 2+ the `command:` is replaced with the API server entrypoint. +# Default `command` is the API server. To run the per-language smoke after +# a rebuild, do: +# docker compose run --rm crafting-table /usr/local/bin/smoke.sh +# +# Volumes mount real Lucy appdata paths so /data + /workspace + /caches +# survive container recreation. Port is bound to LAN only — no Rackham +# proxy. name: crafting-table services: @@ -9,19 +14,27 @@ services: build: . image: crafting-table:local container_name: crafting-table - command: ["/usr/local/bin/smoke.sh"] + command: + - uvicorn + - crafting_table.server:app + - --host + - "0.0.0.0" + - --port + - "8810" user: crafter - working_dir: /workspace + working_dir: /home/crafter + # env_file is optional; copy .env.example to .env to override defaults. + env_file: + - path: .env + required: false + ports: + - "192.168.0.5:8810:8810" volumes: - - workspace:/workspace - - caches:/caches - - data:/data + - /mnt/user/appdata/crafting-table/data:/data + - /mnt/user/appdata/crafting-table/workspace:/workspace + - /mnt/user/appdata/crafting-table/caches:/caches networks: [sulkta] - -volumes: - workspace: - caches: - data: + restart: unless-stopped networks: sulkta: