v0.1 wave 2A (steps 5+6): per-language parsers + findings extraction
- parsers/ package: rust / python / go / typescript / generic
- parser registry with language+recipe -> fallback resolution
- fingerprint hash (kind+file+line+code) for cross-run dedup
- runner.py post-exec hook: parse log, persist findings, count on job row
(extraction runs before mark_job_finished so callers polling on terminal
status see findings_count populated atomically)
- db.insert_finding / list_findings / increment_findings_count DAOs already
shipped in wave 1; wired here
- GET /jobs/{id}/findings now returns real data (server route already
shipped; was returning empty list because nothing populated the table)
- tests/test_parsers/: 6 modules + 11 fixtures (rust/python/go/typescript)
- tests/test_runner_findings.py: 3 integration tests
- README: tick steps 2-6, add Findings section
Suite: 108 passing (62 wave-1 + 46 new).
Spec: memory/spec-crafting-table.md
This commit is contained in:
parent
98306ca2e0
commit
d467b2f5be
30 changed files with 1968 additions and 5 deletions
0
tests/test_parsers/__init__.py
Normal file
0
tests/test_parsers/__init__.py
Normal file
22
tests/test_parsers/conftest.py
Normal file
22
tests/test_parsers/conftest.py
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
"""Local conftest for parser tests — exposes a fixtures-dir helper.
|
||||
|
||||
Parser tests don't need the server reload / TestClient machinery from the
|
||||
top-level conftest; they only need to read fixture files. Keep them light.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
FIXTURES_DIR = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fixtures_dir() -> Path:
|
||||
return FIXTURES_DIR
|
||||
|
||||
|
||||
def load_fixture(*parts: str) -> str:
|
||||
return (FIXTURES_DIR.joinpath(*parts)).read_text(encoding="utf-8")
|
||||
24
tests/test_parsers/fixtures/go/go_vet.json
Normal file
24
tests/test_parsers/fixtures/go/go_vet.json
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
go: downloading example.com/foo v1.2.3
|
||||
{
|
||||
"example.com/foo": {
|
||||
"printf": [
|
||||
{
|
||||
"posn": "/work/foo/main.go:42:9",
|
||||
"message": "Printf format %d has arg name of wrong type string"
|
||||
},
|
||||
{
|
||||
"posn": "/work/foo/util.go:7:5",
|
||||
"message": "Printf format %s has arg n of wrong type int"
|
||||
}
|
||||
],
|
||||
"shadow": [
|
||||
{
|
||||
"posn": "/work/foo/main.go:55:13",
|
||||
"message": "declaration of \"err\" shadows declaration at line 50"
|
||||
}
|
||||
]
|
||||
},
|
||||
"example.com/foo/sub": {
|
||||
"shadow": []
|
||||
}
|
||||
}
|
||||
5
tests/test_parsers/fixtures/go/govulncheck.jsonl
Normal file
5
tests/test_parsers/fixtures/go/govulncheck.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"config":{"protocol_version":"v1.0.0","scanner_name":"govulncheck"}}
|
||||
{"progress":{"message":"Scanning your code…"}}
|
||||
{"finding":{"osv":"GO-2023-1989","module":"golang.org/x/net","fixed_version":"v0.17.0","summary":"HTTP/2 rapid reset can cause excessive work in net/http"}}
|
||||
{"finding":{"osv":"GO-2023-1989","module":"golang.org/x/net","fixed_version":"v0.17.0","summary":"HTTP/2 rapid reset can cause excessive work in net/http"}}
|
||||
{"finding":{"osv":"GO-2024-2611","module":"google.golang.org/protobuf","fixed_version":"v1.33.0","summary":"Infinite loop in github.com/golang/protobuf"}}
|
||||
2
tests/test_parsers/fixtures/python/mypy.jsonl
Normal file
2
tests/test_parsers/fixtures/python/mypy.jsonl
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
{"file":"src/app.py","line":17,"column":4,"severity":"error","message":"Incompatible return value type (got \"str\", expected \"int\")","code":"return-value"}
|
||||
{"file":"src/util.py","line":5,"column":1,"severity":"note","message":"Revealed type is \"builtins.int\"","code":null}
|
||||
20
tests/test_parsers/fixtures/python/pip_audit.json
Normal file
20
tests/test_parsers/fixtures/python/pip_audit.json
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
{
|
||||
"dependencies": [
|
||||
{
|
||||
"name": "requests",
|
||||
"version": "2.20.0",
|
||||
"vulns": [
|
||||
{
|
||||
"id": "PYSEC-2018-28",
|
||||
"fix_versions": ["2.20.1"],
|
||||
"description": "Sensitive Authorization header sent on cross-origin redirect"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "fastapi",
|
||||
"version": "0.95.0",
|
||||
"vulns": []
|
||||
}
|
||||
]
|
||||
}
|
||||
21
tests/test_parsers/fixtures/python/pytest.txt
Normal file
21
tests/test_parsers/fixtures/python/pytest.txt
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
============================= test session starts ==============================
|
||||
collected 5 items
|
||||
|
||||
tests/test_a.py::test_one PASSED [ 20%]
|
||||
tests/test_a.py::test_two FAILED [ 40%]
|
||||
tests/test_b.py::test_three PASSED [ 60%]
|
||||
tests/test_b.py::test_four FAILED [ 80%]
|
||||
tests/test_c.py::test_five PASSED [100%]
|
||||
|
||||
=================================== FAILURES ===================================
|
||||
___________________________________ test_two ___________________________________
|
||||
assert False
|
||||
E AssertionError
|
||||
__________________________________ test_four ___________________________________
|
||||
assert 1 == 2
|
||||
E AssertionError
|
||||
|
||||
=========================== short test summary info ============================
|
||||
FAILED tests/test_a.py::test_two - AssertionError
|
||||
FAILED tests/test_b.py::test_four - AssertionError: assert 1 == 2
|
||||
========================= 2 failed, 3 passed in 0.12s ==========================
|
||||
16
tests/test_parsers/fixtures/python/ruff.json
Normal file
16
tests/test_parsers/fixtures/python/ruff.json
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
[
|
||||
{
|
||||
"code": "F401",
|
||||
"message": "`os` imported but unused",
|
||||
"filename": "/work/src/app.py",
|
||||
"location": {"row": 3, "column": 1},
|
||||
"end_location": {"row": 3, "column": 10},
|
||||
"fix": {"applicability": "safe", "message": "Remove unused import"}
|
||||
},
|
||||
{
|
||||
"code": "E501",
|
||||
"message": "Line too long (102 > 88 characters)",
|
||||
"filename": "/work/src/app.py",
|
||||
"location": {"row": 42, "column": 89}
|
||||
}
|
||||
]
|
||||
2
tests/test_parsers/fixtures/rust/cargo_audit.json
Normal file
2
tests/test_parsers/fixtures/rust/cargo_audit.json
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
cargo audit fetched advisory database from https://github.com/RustSec/advisory-db
|
||||
{"database":{"advisory_count":634},"lockfile":{"dependency_count":120},"settings":{},"vulnerabilities":{"found":true,"count":2,"list":[{"advisory":{"id":"RUSTSEC-2024-0123","title":"openssl: Use-after-free in SslContextBuilder","description":"Affected versions of this crate may use freed memory when…","date":"2024-08-12","aliases":["CVE-2024-12345"],"keywords":["use-after-free","openssl"],"categories":["memory-corruption"]},"versions":{"patched":[">=0.10.66"],"unaffected":[]},"affected":null,"package":{"name":"openssl","version":"0.10.55","source":"registry+https://github.com/rust-lang/crates.io-index","checksum":"abc123","dependencies":[]}},{"advisory":{"id":"RUSTSEC-2024-0099","title":"time: Out-of-bounds read in parse","description":"The time crate had an OOB read…","date":"2024-04-01","aliases":[],"keywords":["oob"],"categories":["denial-of-service"]},"versions":{"patched":[],"unaffected":[]},"affected":null,"package":{"name":"time","version":"0.2.27","source":"registry+https://github.com/rust-lang/crates.io-index","checksum":"def456","dependencies":[]}}]},"warnings":{}}
|
||||
5
tests/test_parsers/fixtures/rust/cargo_clippy.jsonl
Normal file
5
tests/test_parsers/fixtures/rust/cargo_clippy.jsonl
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
{"reason":"compiler-artifact","package_id":"foo 0.1.0","target":{"name":"foo"}}
|
||||
{"reason":"compiler-message","package_id":"foo 0.1.0","target":{"name":"foo"},"message":{"rendered":"warning: unused variable: `x`\n --> src/lib.rs:12:9","level":"warning","code":{"code":"unused_variables","explanation":null},"message":"unused variable: `x`","spans":[{"file_name":"src/lib.rs","line_start":12,"line_end":12,"column_start":9,"column_end":10,"is_primary":true}],"children":[{"rendered":"help: if this is intentional, prefix it with an underscore: `_x`","level":"help","message":"if this is intentional, prefix it with an underscore","spans":[]}]}}
|
||||
{"reason":"compiler-message","package_id":"foo 0.1.0","target":{"name":"foo"},"message":{"rendered":"error[E0382]: borrow of moved value: `s`","level":"error","code":{"code":"E0382","explanation":null},"message":"borrow of moved value: `s`","spans":[{"file_name":"src/main.rs","line_start":42,"line_end":42,"column_start":5,"column_end":10,"is_primary":true}],"children":[]}}
|
||||
{"reason":"compiler-message","package_id":"foo 0.1.0","target":{"name":"foo"},"message":{"rendered":"note: ...","level":"note","code":null,"message":"note: nothing","spans":[],"children":[]}}
|
||||
{"reason":"build-finished","success":true}
|
||||
23
tests/test_parsers/fixtures/rust/cargo_test.txt
Normal file
23
tests/test_parsers/fixtures/rust/cargo_test.txt
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
Finished test [unoptimized + debuginfo] target(s) in 0.45s
|
||||
Running unittests src/lib.rs (target/debug/deps/foo-abc)
|
||||
|
||||
running 4 tests
|
||||
test math::tests::adds_two ... ok
|
||||
test math::tests::adds_negative ... FAILED
|
||||
test parser::tests::parses_empty ... ok
|
||||
test parser::tests::parses_garbage ... FAILED
|
||||
|
||||
failures:
|
||||
|
||||
---- math::tests::adds_negative stdout ----
|
||||
thread 'math::tests::adds_negative' panicked at 'assertion failed', src/math.rs:14:5
|
||||
|
||||
---- parser::tests::parses_garbage stdout ----
|
||||
thread 'parser::tests::parses_garbage' panicked at 'expected Err', src/parser.rs:33:5
|
||||
|
||||
|
||||
failures:
|
||||
math::tests::adds_negative
|
||||
parser::tests::parses_garbage
|
||||
|
||||
test result: FAILED. 2 passed; 2 failed; 0 ignored; 0 measured; 0 filtered out
|
||||
30
tests/test_parsers/fixtures/typescript/eslint.json
Normal file
30
tests/test_parsers/fixtures/typescript/eslint.json
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
[
|
||||
{
|
||||
"filePath": "/work/src/index.ts",
|
||||
"messages": [
|
||||
{
|
||||
"ruleId": "no-unused-vars",
|
||||
"severity": 1,
|
||||
"message": "'foo' is defined but never used.",
|
||||
"line": 5,
|
||||
"column": 7,
|
||||
"nodeType": "Identifier"
|
||||
},
|
||||
{
|
||||
"ruleId": "@typescript-eslint/no-explicit-any",
|
||||
"severity": 2,
|
||||
"message": "Unexpected any. Specify a different type.",
|
||||
"line": 12,
|
||||
"column": 18
|
||||
}
|
||||
],
|
||||
"errorCount": 1,
|
||||
"warningCount": 1
|
||||
},
|
||||
{
|
||||
"filePath": "/work/src/util.ts",
|
||||
"messages": [],
|
||||
"errorCount": 0,
|
||||
"warningCount": 0
|
||||
}
|
||||
]
|
||||
5
tests/test_parsers/fixtures/typescript/tsc.txt
Normal file
5
tests/test_parsers/fixtures/typescript/tsc.txt
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
src/index.ts(5,7): error TS2304: Cannot find name 'foo'.
|
||||
src/index.ts(12,18): error TS7006: Parameter 'x' implicitly has an 'any' type.
|
||||
src/util.ts(3,1): warning TS6133: 'unused' is declared but its value is never read.
|
||||
|
||||
Found 2 errors and 1 warning in 2 files.
|
||||
64
tests/test_parsers/test_fingerprint.py
Normal file
64
tests/test_parsers/test_fingerprint.py
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
"""Fingerprint helper unit tests.
|
||||
|
||||
Properties asserted:
|
||||
- determinism (same args → same hash)
|
||||
- locator-only (changing message DOESN'T change fingerprint)
|
||||
- file/line/code each contribute (changing any of them DOES change it)
|
||||
- 16-char output, hex
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from crafting_table.parsers.base import fingerprint
|
||||
|
||||
|
||||
def test_fingerprint_deterministic():
|
||||
a = fingerprint("lint", "src/x.py", 10, "F401", "unused import os")
|
||||
b = fingerprint("lint", "src/x.py", 10, "F401", "unused import os")
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_fingerprint_message_excluded():
|
||||
"""Tool wording drifts; message must NOT contribute to the hash."""
|
||||
a = fingerprint("lint", "src/x.py", 10, "F401", "unused import os")
|
||||
b = fingerprint("lint", "src/x.py", 10, "F401", "wholly different wording")
|
||||
assert a == b
|
||||
|
||||
|
||||
def test_fingerprint_file_changes_hash():
|
||||
a = fingerprint("lint", "src/x.py", 10, "F401", "msg")
|
||||
b = fingerprint("lint", "src/y.py", 10, "F401", "msg")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_fingerprint_line_changes_hash():
|
||||
a = fingerprint("lint", "src/x.py", 10, "F401", "msg")
|
||||
b = fingerprint("lint", "src/x.py", 11, "F401", "msg")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_fingerprint_code_changes_hash():
|
||||
a = fingerprint("lint", "src/x.py", 10, "F401", "msg")
|
||||
b = fingerprint("lint", "src/x.py", 10, "E501", "msg")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_fingerprint_kind_changes_hash():
|
||||
a = fingerprint("lint", "src/x.py", 10, "F401", "msg")
|
||||
b = fingerprint("cve", "src/x.py", 10, "F401", "msg")
|
||||
assert a != b
|
||||
|
||||
|
||||
def test_fingerprint_handles_none_locator_parts():
|
||||
# Findings without file/line (e.g. CVEs) still get a deterministic hash.
|
||||
a = fingerprint("cve", None, None, "RUSTSEC-2024-0123", "openssl bad")
|
||||
b = fingerprint("cve", None, None, "RUSTSEC-2024-0123", "openssl bad")
|
||||
assert a == b
|
||||
assert a != fingerprint("cve", None, None, "RUSTSEC-2024-0124", "openssl bad")
|
||||
|
||||
|
||||
def test_fingerprint_shape():
|
||||
fp = fingerprint("lint", "x", 1, "C", "m")
|
||||
assert len(fp) == 16
|
||||
assert re.fullmatch(r"[0-9a-f]{16}", fp)
|
||||
36
tests/test_parsers/test_generic.py
Normal file
36
tests/test_parsers/test_generic.py
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
"""GenericParser fallback unit tests."""
|
||||
from __future__ import annotations
|
||||
|
||||
from crafting_table.parsers.generic import GenericParser
|
||||
from crafting_table.parsers import find_parser
|
||||
|
||||
|
||||
def test_generic_zero_exit_no_findings():
|
||||
out = GenericParser.parse("any output", 0, "build")
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_generic_nonzero_exit_emits_one_finding():
|
||||
out = GenericParser.parse("oops", 1, "build")
|
||||
assert len(out) == 1
|
||||
f = out[0]
|
||||
assert f.kind == "recipe_fail"
|
||||
assert f.severity == "warn"
|
||||
assert f.code == "exit_1"
|
||||
assert "build" in f.message
|
||||
assert "1" in f.message
|
||||
|
||||
|
||||
def test_generic_matches_anything():
|
||||
assert GenericParser.matches("anylang", "anyrecipe") is True
|
||||
|
||||
|
||||
def test_registry_falls_back_to_generic_for_unknown_lang():
|
||||
cls = find_parser("ruby", "audit")
|
||||
assert cls is GenericParser
|
||||
|
||||
|
||||
def test_registry_falls_back_to_generic_for_unknown_recipe():
|
||||
# Rust parser declines unknown recipes; resolver should drop to generic.
|
||||
cls = find_parser("rust", "deploy")
|
||||
assert cls is GenericParser
|
||||
57
tests/test_parsers/test_go.py
Normal file
57
tests/test_parsers/test_go.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
"""GoParser unit tests — go vet + govulncheck."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .conftest import load_fixture
|
||||
from crafting_table.parsers.go import GoParser, _parse_posn
|
||||
|
||||
|
||||
def test_go_vet_extracts_diagnostics():
|
||||
raw = load_fixture("go", "go_vet.json")
|
||||
findings = GoParser.parse(raw, exit_code=1, recipe="lint")
|
||||
# 2 printf + 1 shadow = 3.
|
||||
assert len(findings) == 3
|
||||
|
||||
by_code: dict[str, int] = {}
|
||||
for f in findings:
|
||||
assert f.kind == "lint"
|
||||
by_code[f.code] = by_code.get(f.code, 0) + 1
|
||||
assert by_code["printf"] == 2
|
||||
assert by_code["shadow"] == 1
|
||||
|
||||
printf_first = next(f for f in findings if f.code == "printf")
|
||||
assert printf_first.file == "/work/foo/main.go"
|
||||
assert printf_first.line == 42
|
||||
|
||||
|
||||
def test_go_vet_garbage_no_findings():
|
||||
findings = GoParser.parse("nothing useful here", exit_code=0, recipe="lint")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_govulncheck_dedups_by_osv_id():
|
||||
raw = load_fixture("go", "govulncheck.jsonl")
|
||||
findings = GoParser.parse(raw, exit_code=3, recipe="audit")
|
||||
# 3 finding records → 2 unique OSV ids.
|
||||
assert len(findings) == 2
|
||||
ids = sorted(f.code for f in findings)
|
||||
assert ids == ["GO-2023-1989", "GO-2024-2611"]
|
||||
f0 = next(f for f in findings if f.code == "GO-2023-1989")
|
||||
assert f0.kind == "cve"
|
||||
assert f0.suggested_fix and "v0.17.0" in f0.suggested_fix
|
||||
assert f0.extras["package"] == "golang.org/x/net"
|
||||
|
||||
|
||||
def test_go_audit_clean_log_no_findings():
|
||||
findings = GoParser.parse('{"config":{"protocol_version":"v1.0.0"}}', exit_code=0, recipe="audit")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_go_build_recipe_falls_through():
|
||||
f = GoParser.parse("any", exit_code=1, recipe="build")
|
||||
assert len(f) == 1 and f[0].kind == "recipe_fail"
|
||||
|
||||
|
||||
def test_parse_posn_helper():
|
||||
assert _parse_posn("/abs/x.go:42:9") == ("/abs/x.go", 42)
|
||||
assert _parse_posn("rel/y.go:7") == ("rel/y.go", 7)
|
||||
assert _parse_posn("") == (None, None)
|
||||
91
tests/test_parsers/test_python.py
Normal file
91
tests/test_parsers/test_python.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
"""PythonParser unit tests — ruff + mypy + pip-audit + pytest."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .conftest import load_fixture
|
||||
from crafting_table.parsers.python import PythonParser
|
||||
|
||||
|
||||
def test_python_lint_ruff_array():
|
||||
raw = load_fixture("python", "ruff.json")
|
||||
findings = PythonParser.parse(raw, exit_code=1, recipe="lint")
|
||||
# 2 ruff entries → 2 findings.
|
||||
assert len(findings) == 2
|
||||
|
||||
by_code = {f.code: f for f in findings}
|
||||
assert "F401" in by_code and "E501" in by_code
|
||||
|
||||
f401 = by_code["F401"]
|
||||
assert f401.kind == "lint"
|
||||
assert f401.severity == "warn"
|
||||
assert f401.file == "/work/src/app.py"
|
||||
assert f401.line == 3
|
||||
# ruff fix.message should map into suggested_fix
|
||||
assert f401.suggested_fix is not None
|
||||
|
||||
|
||||
def test_python_lint_mypy_jsonl():
|
||||
raw = load_fixture("python", "mypy.jsonl")
|
||||
findings = PythonParser.parse(raw, exit_code=1, recipe="lint")
|
||||
# 2 mypy lines: 1 error (kept), 1 note (still parsed but warn).
|
||||
assert len(findings) == 2
|
||||
err = next(f for f in findings if f.severity == "error")
|
||||
assert err.code == "return-value"
|
||||
assert err.file == "src/app.py"
|
||||
assert err.line == 17
|
||||
|
||||
|
||||
def test_python_lint_handles_garbage():
|
||||
findings = PythonParser.parse("oops not json", exit_code=1, recipe="lint")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_python_audit_pip_audit():
|
||||
raw = load_fixture("python", "pip_audit.json")
|
||||
findings = PythonParser.parse(raw, exit_code=1, recipe="audit")
|
||||
# Only requests has a vuln; fastapi has none.
|
||||
assert len(findings) == 1
|
||||
f = findings[0]
|
||||
assert f.kind == "cve"
|
||||
assert f.severity == "high"
|
||||
assert f.code == "PYSEC-2018-28"
|
||||
assert "requests" in f.message
|
||||
assert f.suggested_fix == "bump requests to 2.20.1"
|
||||
assert f.extras["package"] == "requests"
|
||||
|
||||
|
||||
def test_python_audit_clean_log_no_findings():
|
||||
raw = '{"dependencies":[]}'
|
||||
findings = PythonParser.parse(raw, exit_code=0, recipe="audit")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_python_test_parses_failed_lines():
|
||||
raw = load_fixture("python", "pytest.txt")
|
||||
findings = PythonParser.parse(raw, exit_code=1, recipe="test")
|
||||
assert len(findings) == 2
|
||||
codes = sorted(f.code for f in findings)
|
||||
assert codes == sorted(["tests/test_a.py::test_two", "tests/test_b.py::test_four"])
|
||||
for f in findings:
|
||||
assert f.kind == "test_fail"
|
||||
assert f.severity == "error"
|
||||
assert f.file is not None
|
||||
assert f.file.endswith(".py")
|
||||
|
||||
|
||||
def test_python_test_zero_exit_no_findings():
|
||||
findings = PythonParser.parse("all passed", exit_code=0, recipe="test")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_python_test_nonzero_no_failed_marker_emits_synthetic():
|
||||
findings = PythonParser.parse("collection error", exit_code=2, recipe="test")
|
||||
assert len(findings) == 1
|
||||
assert findings[0].kind == "test_fail"
|
||||
assert "exit_2" in findings[0].code
|
||||
|
||||
|
||||
def test_python_matches():
|
||||
assert PythonParser.matches("python", "lint")
|
||||
assert PythonParser.matches("python", "audit")
|
||||
assert PythonParser.matches("python", "test")
|
||||
assert not PythonParser.matches("rust", "lint")
|
||||
108
tests/test_parsers/test_rust.py
Normal file
108
tests/test_parsers/test_rust.py
Normal file
|
|
@ -0,0 +1,108 @@
|
|||
"""RustParser unit tests — driven from fixtures/rust/ samples."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .conftest import load_fixture
|
||||
from crafting_table.parsers.rust import RustParser
|
||||
|
||||
|
||||
def test_rust_audit_extracts_two_cves():
|
||||
raw = load_fixture("rust", "cargo_audit.json")
|
||||
findings = RustParser.parse(raw, exit_code=1, recipe="audit")
|
||||
assert len(findings) == 2
|
||||
|
||||
f1 = findings[0]
|
||||
assert f1.kind == "cve"
|
||||
assert f1.code == "RUSTSEC-2024-0123"
|
||||
assert f1.severity == "high"
|
||||
assert "openssl" in f1.message
|
||||
assert "0.10.55" in f1.message
|
||||
assert "0.10.66" in f1.message
|
||||
assert f1.suggested_fix is not None
|
||||
assert "0.10.66" in f1.suggested_fix
|
||||
assert f1.raw_json is not None
|
||||
assert f1.extras["package"] == "openssl"
|
||||
|
||||
f2 = findings[1]
|
||||
assert f2.code == "RUSTSEC-2024-0099"
|
||||
# No patched versions → no suggested_fix
|
||||
assert f2.suggested_fix is None
|
||||
|
||||
|
||||
def test_rust_audit_clean_log_no_findings():
|
||||
# No vulnerabilities in the envelope.
|
||||
raw = '{"vulnerabilities":{"found":false,"count":0,"list":[]}}'
|
||||
findings = RustParser.parse(raw, exit_code=0, recipe="audit")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_rust_audit_garbage_log_no_findings():
|
||||
findings = RustParser.parse("not json at all", exit_code=1, recipe="audit")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_rust_clippy_extracts_warning_and_error():
|
||||
raw = load_fixture("rust", "cargo_clippy.jsonl")
|
||||
findings = RustParser.parse(raw, exit_code=1, recipe="lint")
|
||||
# Two compiler-message lines with level in {warning, error}; the "note"
|
||||
# one should be filtered out.
|
||||
assert len(findings) == 2
|
||||
|
||||
by_code = {f.code: f for f in findings}
|
||||
assert "unused_variables" in by_code
|
||||
assert "E0382" in by_code
|
||||
|
||||
w = by_code["unused_variables"]
|
||||
assert w.severity == "warn"
|
||||
assert w.kind == "lint"
|
||||
assert w.file == "src/lib.rs"
|
||||
assert w.line == 12
|
||||
assert w.suggested_fix is not None
|
||||
assert "_x" in w.suggested_fix
|
||||
|
||||
e = by_code["E0382"]
|
||||
assert e.severity == "error"
|
||||
assert e.line == 42
|
||||
assert e.file == "src/main.rs"
|
||||
|
||||
|
||||
def test_rust_clippy_skips_non_compiler_message_lines():
|
||||
raw = '{"reason":"build-finished","success":true}\n'
|
||||
findings = RustParser.parse(raw, exit_code=0, recipe="lint")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_rust_test_parses_failures():
|
||||
raw = load_fixture("rust", "cargo_test.txt")
|
||||
findings = RustParser.parse(raw, exit_code=101, recipe="test")
|
||||
codes = sorted(f.code for f in findings)
|
||||
assert codes == sorted(["math::tests::adds_negative", "parser::tests::parses_garbage"])
|
||||
for f in findings:
|
||||
assert f.kind == "test_fail"
|
||||
assert f.severity == "error"
|
||||
|
||||
|
||||
def test_rust_test_zero_exit_no_findings():
|
||||
findings = RustParser.parse("test result: ok. all passed", exit_code=0, recipe="test")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_rust_test_nonzero_no_failed_lines_emits_synthetic():
|
||||
findings = RustParser.parse("compile error", exit_code=2, recipe="test")
|
||||
assert len(findings) == 1
|
||||
assert findings[0].kind == "test_fail"
|
||||
assert "exit_2" in findings[0].code
|
||||
|
||||
|
||||
def test_rust_build_recipe_falls_through_to_recipe_fail():
|
||||
findings = RustParser.parse("anything", exit_code=1, recipe="build")
|
||||
assert len(findings) == 1
|
||||
assert findings[0].kind == "recipe_fail"
|
||||
|
||||
|
||||
def test_rust_matches_only_rust_recipes():
|
||||
assert RustParser.matches("rust", "audit")
|
||||
assert RustParser.matches("rust", "lint")
|
||||
assert RustParser.matches("rust", "test")
|
||||
assert RustParser.matches("rust", "build")
|
||||
assert not RustParser.matches("python", "audit")
|
||||
assert not RustParser.matches("rust", "deploy")
|
||||
56
tests/test_parsers/test_typescript.py
Normal file
56
tests/test_parsers/test_typescript.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""TypeScriptParser unit tests — eslint + tsc."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .conftest import load_fixture
|
||||
from crafting_table.parsers.typescript import TypeScriptParser
|
||||
|
||||
|
||||
def test_eslint_array_two_messages():
|
||||
raw = load_fixture("typescript", "eslint.json")
|
||||
findings = TypeScriptParser.parse(raw, exit_code=1, recipe="lint")
|
||||
# First file has 2 messages, second has 0.
|
||||
assert len(findings) == 2
|
||||
|
||||
by_code = {f.code: f for f in findings}
|
||||
assert "no-unused-vars" in by_code
|
||||
assert "@typescript-eslint/no-explicit-any" in by_code
|
||||
|
||||
warn = by_code["no-unused-vars"]
|
||||
assert warn.severity == "warn"
|
||||
assert warn.line == 5
|
||||
assert warn.file == "/work/src/index.ts"
|
||||
|
||||
err = by_code["@typescript-eslint/no-explicit-any"]
|
||||
assert err.severity == "error"
|
||||
|
||||
|
||||
def test_tsc_human_output_parses():
|
||||
raw = load_fixture("typescript", "tsc.txt")
|
||||
findings = TypeScriptParser.parse(raw, exit_code=1, recipe="lint")
|
||||
# 2 errors + 1 warning.
|
||||
assert len(findings) == 3
|
||||
codes = sorted(f.code for f in findings)
|
||||
assert codes == ["TS2304", "TS6133", "TS7006"]
|
||||
err = next(f for f in findings if f.code == "TS2304")
|
||||
assert err.severity == "error"
|
||||
assert err.file == "src/index.ts"
|
||||
assert err.line == 5
|
||||
|
||||
|
||||
def test_typescript_lint_garbage_no_findings():
|
||||
findings = TypeScriptParser.parse("nothing", exit_code=0, recipe="lint")
|
||||
assert findings == []
|
||||
|
||||
|
||||
def test_javascript_alias_handled_by_typescript_parser():
|
||||
# The registry routes "javascript" to TypeScriptParser.
|
||||
from crafting_table.parsers import find_parser
|
||||
from crafting_table.parsers.typescript import TypeScriptParser as TSP
|
||||
|
||||
assert find_parser("javascript", "lint") is TSP
|
||||
|
||||
|
||||
def test_ts_audit_falls_through_to_recipe_fail():
|
||||
f = TypeScriptParser.parse("any", exit_code=1, recipe="audit")
|
||||
assert len(f) == 1
|
||||
assert f[0].kind == "recipe_fail"
|
||||
196
tests/test_runner_findings.py
Normal file
196
tests/test_runner_findings.py
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
"""Integration test — runner runs a recipe whose output is parser-shaped, and
|
||||
findings end up in the DB + visible via GET /jobs/{id}/findings.
|
||||
|
||||
We use python:lint with a stub command that emits a known-good ruff JSON
|
||||
array, then verify:
|
||||
- findings_count on the job row reflects the parsed entries
|
||||
- GET /jobs/{id}/findings returns rows with the right kind/code/file/line
|
||||
- fingerprint is populated and stable across rows for the same locator
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import sample_project_payload
|
||||
|
||||
|
||||
def _make_local_git_repo(root: Path) -> str:
|
||||
if shutil.which("git") is None:
|
||||
pytest.skip("git binary not present in test environment")
|
||||
repo = root / "findings-fixture-repo"
|
||||
repo.mkdir()
|
||||
subprocess.run(["git", "init", "-q", "-b", "main"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "user.email", "test@example"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "user.name", "test"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "config", "commit.gpgsign", "false"], cwd=repo, check=True)
|
||||
(repo / "README.md").write_text("hello\n")
|
||||
subprocess.run(["git", "add", "README.md"], cwd=repo, check=True)
|
||||
subprocess.run(["git", "commit", "-q", "-m", "initial"], cwd=repo, check=True)
|
||||
return str(repo)
|
||||
|
||||
|
||||
def _wait_terminal(tc, bearer: str, job_id: str, timeout: float = 30.0) -> dict:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
r = tc.get(
|
||||
f"/jobs/{job_id}",
|
||||
headers={"Authorization": f"Bearer {bearer}"},
|
||||
)
|
||||
body = r.json()
|
||||
if body["job"]["status"] in ("succeeded", "failed", "timed_out", "cancelled"):
|
||||
return body
|
||||
time.sleep(0.1)
|
||||
raise AssertionError("job never finished")
|
||||
|
||||
|
||||
def test_python_lint_findings_persisted_via_runner(client, tmp_path):
|
||||
"""python:lint with a ruff-shaped JSON stub → 2 findings persisted."""
|
||||
tc, ctx = client
|
||||
git_url = _make_local_git_repo(tmp_path)
|
||||
|
||||
# Ruff-shaped stub. echo + exit 1 (lint findings → non-zero exit) so
|
||||
# the recipe terminates the way ruff really would.
|
||||
ruff_stub = json.dumps(
|
||||
[
|
||||
{
|
||||
"code": "F401",
|
||||
"message": "'os' imported but unused",
|
||||
"filename": "src/app.py",
|
||||
"location": {"row": 3, "column": 1},
|
||||
},
|
||||
{
|
||||
"code": "E501",
|
||||
"message": "Line too long",
|
||||
"filename": "src/app.py",
|
||||
"location": {"row": 42, "column": 89},
|
||||
},
|
||||
]
|
||||
)
|
||||
# Single-quote the JSON so the shell doesn't interpret double-quotes.
|
||||
lint_cmd = f"echo '{ruff_stub}'; exit 1"
|
||||
|
||||
payload = sample_project_payload(name="ct-findings-py")
|
||||
payload["git_url"] = git_url
|
||||
payload["subprojects"][0]["language"] = "python"
|
||||
payload["subprojects"][0]["lint"] = lint_cmd
|
||||
payload["subprojects"][0]["timeout_secs"] = 20
|
||||
|
||||
r = tc.post(
|
||||
"/projects",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
json=payload,
|
||||
)
|
||||
assert r.status_code == 200, r.text
|
||||
|
||||
r2 = tc.post(
|
||||
"/projects/ct-findings-py/jobs",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
json={"recipe": "lint"},
|
||||
)
|
||||
assert r2.status_code == 200, r2.text
|
||||
job_id = r2.json()["job_id"]
|
||||
|
||||
final = _wait_terminal(tc, ctx["alpha_bearer"], job_id)
|
||||
# Recipe exited 1 so the job is "failed" — but parsing still happens.
|
||||
assert final["job"]["status"] == "failed"
|
||||
assert final["job"]["exit_code"] == 1
|
||||
assert final["job"]["findings_count"] == 2
|
||||
|
||||
r3 = tc.get(
|
||||
f"/jobs/{job_id}/findings",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
)
|
||||
assert r3.status_code == 200, r3.text
|
||||
findings = r3.json()["findings"]
|
||||
assert len(findings) == 2
|
||||
by_code = {f["code"]: f for f in findings}
|
||||
assert "F401" in by_code and "E501" in by_code
|
||||
f401 = by_code["F401"]
|
||||
assert f401["kind"] == "lint"
|
||||
assert f401["severity"] == "warn"
|
||||
assert f401["file"] == "src/app.py"
|
||||
assert f401["line"] == 3
|
||||
assert f401["fingerprint"]
|
||||
assert len(f401["fingerprint"]) == 16
|
||||
|
||||
|
||||
def test_unknown_lang_recipe_falls_back_to_generic(client, tmp_path):
|
||||
"""A recipe with no parser registered emits exactly one recipe_fail
|
||||
finding when it exits non-zero, and zero findings when it exits 0."""
|
||||
tc, ctx = client
|
||||
git_url = _make_local_git_repo(tmp_path)
|
||||
|
||||
# Use `ruby` which has no parser; our PythonParser etc. all decline.
|
||||
payload = sample_project_payload(name="ct-findings-generic")
|
||||
payload["git_url"] = git_url
|
||||
payload["subprojects"][0]["language"] = "ruby"
|
||||
payload["subprojects"][0]["audit"] = "echo audit-output; exit 5"
|
||||
payload["subprojects"][0]["timeout_secs"] = 20
|
||||
|
||||
r = tc.post(
|
||||
"/projects",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
json=payload,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
|
||||
r2 = tc.post(
|
||||
"/projects/ct-findings-generic/jobs",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
json={"recipe": "audit"},
|
||||
)
|
||||
job_id = r2.json()["job_id"]
|
||||
final = _wait_terminal(tc, ctx["alpha_bearer"], job_id)
|
||||
assert final["job"]["status"] == "failed"
|
||||
assert final["job"]["findings_count"] == 1
|
||||
|
||||
r3 = tc.get(
|
||||
f"/jobs/{job_id}/findings",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
)
|
||||
rows = r3.json()["findings"]
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["kind"] == "recipe_fail"
|
||||
assert rows[0]["code"] == "exit_5"
|
||||
|
||||
|
||||
def test_clean_recipe_produces_zero_findings(client, tmp_path):
|
||||
"""Successful run with no parseable signal → no findings rows, count=0."""
|
||||
tc, ctx = client
|
||||
git_url = _make_local_git_repo(tmp_path)
|
||||
|
||||
payload = sample_project_payload(name="ct-findings-clean")
|
||||
payload["git_url"] = git_url
|
||||
payload["subprojects"][0]["language"] = "python"
|
||||
# Empty ruff JSON array → 0 findings, exit 0.
|
||||
payload["subprojects"][0]["lint"] = "echo '[]'; exit 0"
|
||||
payload["subprojects"][0]["timeout_secs"] = 20
|
||||
|
||||
r = tc.post(
|
||||
"/projects",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
json=payload,
|
||||
)
|
||||
assert r.status_code == 200
|
||||
|
||||
r2 = tc.post(
|
||||
"/projects/ct-findings-clean/jobs",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
json={"recipe": "lint"},
|
||||
)
|
||||
job_id = r2.json()["job_id"]
|
||||
final = _wait_terminal(tc, ctx["alpha_bearer"], job_id)
|
||||
assert final["job"]["status"] == "succeeded"
|
||||
assert final["job"]["findings_count"] == 0
|
||||
|
||||
r3 = tc.get(
|
||||
f"/jobs/{job_id}/findings",
|
||||
headers={"Authorization": f"Bearer {ctx['alpha_bearer']}"},
|
||||
)
|
||||
assert r3.json()["findings"] == []
|
||||
Loading…
Add table
Add a link
Reference in a new issue