Fix the budget gate and add CI, lint and coverage guards
- check_budget.py escalated a hard index-row violation to 'warn' on both branches, so INDEX_ROW_HARD could never fail anything. It fails now, and tests/unit/test_budget_measurement.py pins both thresholds. - The corpus note counted scripts/__pycache__/*.pyc, so the same commit reported 27 scripts after the suite had run and 16 on a clean checkout. - TOKENS_PER_CHAR is re-derived from a measurement (26,795 chars -> 6,065 tokens = 4.418); the comment's old figure and its arithmetic were both wrong for this file. - check_repo.py's yaml sentinel carries the type ignore, so mypy is clean with PyYAML installed as well as without it. - The CPUID test now skips with the probe's own note on a host without CPUID instead of failing, and pytest.ini points at tests/README.md, which exists, instead of an EXTENSION file that never did. Tooling: .flake8 (a measured 120-column limit), .gitattributes, .gitignore and .github/workflows/ci.yml (the five gates, the suite on 3.11-3.14 and a lint job), which is what CONTRIBUTING.md already claimed CI runs. The pull-request template it refers to now exists. Content: pitfalls P26-P34 -- the analysis-level failures the catalogue was missing (sampling read as observation, a self-built parser trusted, a verification that could not have failed, a shell-mangled command line) -- registered in the symptom index and its routing mirror. FINDINGS F11-F15 record the defects above. Tests: the coverage guard derives its subject from scripts/ rather than a hand-written list, pe_triage.py and pe_diff.py gain their first tests, and plan.py / preflight.py are declared uncovered with reasons.
This commit is contained in:
@@ -0,0 +1,80 @@
|
||||
"""Coverage of the shipped scripts, derived from the directory rather than a hand-written list.
|
||||
|
||||
Borrowed mechanism: the sibling apk-reverse repository derives its CLI contract cases from
|
||||
skills/*/scripts/*.py and from each script's own --help output, so a new script is covered the day
|
||||
it lands instead of the day someone remembers to add it. What is *not* borrowed is that repository's
|
||||
single 0-4 exit-code table: this kit publishes a per-script vocabulary (pepatch.py uses 3 for "not a
|
||||
parseable PE", where the table means "the environment lacks a capability"), and a test that rewrote
|
||||
those meanings would assert a policy this repository deliberately does not have.
|
||||
|
||||
The guard here is therefore narrower, and it is about *coverage*, not exit codes:
|
||||
|
||||
* every shipped .py is either in conftest.SCRIPT_FILES (some layer of this suite exercises it) or
|
||||
named in DECLARED_UNCOVERED together with the reason it is not;
|
||||
* every declared gap carries a real sentence, because "TODO" is how a gap becomes permanent;
|
||||
* every shipped script -- covered or declared -- answers --help with its usage block, which is the
|
||||
cheapest possible exercise of its argparse table and its import path.
|
||||
|
||||
A new script fails this file until one of those lists changes. That is the point: the alternative
|
||||
is a suite that looks complete while silently ignoring half the kit.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import SCRIPTS_DIR, SCRIPT_FILES
|
||||
|
||||
pytestmark = pytest.mark.cli
|
||||
|
||||
# Scripts no layer exercises yet. Each entry is a recorded gap, not an exemption: the reason says
|
||||
# what would be needed to close it, so the next pass can decide rather than rediscover.
|
||||
DECLARED_UNCOVERED = {
|
||||
"plan.py":
|
||||
"composes capabilities.py and doctor.py into a route from a stated intent. Both halves have "
|
||||
"their own tests; the composition reads live host facts, so a fixture that pinned its output "
|
||||
"would pin this machine's capability set instead of the mapping. Closing it needs a stubbed "
|
||||
"probe, and that stub does not exist yet.",
|
||||
"preflight.py":
|
||||
"answers session readiness from live host facts (free space, route prerequisites) plus one "
|
||||
"sample. The sample half could be pinned to a generated PE; the host half has no generator, "
|
||||
"and asserting on the runner's environment would encode this machine as an expectation.",
|
||||
}
|
||||
|
||||
|
||||
def shipped_scripts():
|
||||
"""Every runnable python script the skill ships, sorted."""
|
||||
return sorted(p.name for p in SCRIPTS_DIR.glob("*.py"))
|
||||
|
||||
|
||||
def test_every_shipped_script_is_exercised_or_declared():
|
||||
"""A script nobody tests must at least be a script somebody decided not to test."""
|
||||
shipped = set(shipped_scripts())
|
||||
assert shipped, "no scripts found under %s" % SCRIPTS_DIR
|
||||
silently_uncovered = shipped - set(SCRIPT_FILES) - set(DECLARED_UNCOVERED)
|
||||
assert not silently_uncovered, (
|
||||
"these scripts are shipped but neither exercised nor declared uncovered: %s. Add a test and "
|
||||
"register the script in conftest.SCRIPT_FILES, or add it to DECLARED_UNCOVERED with the "
|
||||
"reason." % sorted(silently_uncovered))
|
||||
|
||||
|
||||
def test_declared_entries_are_neither_stale_nor_hollow():
|
||||
"""A gap that was closed must leave the list; a gap with no reason is a hidden one."""
|
||||
both = set(SCRIPT_FILES) & set(DECLARED_UNCOVERED)
|
||||
assert not both, ("these scripts are in SCRIPT_FILES and also declared uncovered: %s -- the "
|
||||
"declaration is stale" % sorted(both))
|
||||
for name, reason in DECLARED_UNCOVERED.items():
|
||||
assert (SCRIPTS_DIR / name).is_file(), "%s is declared uncovered but does not exist" % name
|
||||
assert len(reason.split()) >= 12, (
|
||||
"%s: the reason must say what would close the gap, not just mark it" % name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("script", shipped_scripts())
|
||||
def test_every_shipped_script_answers_help_with_usage(script):
|
||||
"""The usage block is the script's published contract; it must render on every host."""
|
||||
proc = subprocess.run([sys.executable, str(SCRIPTS_DIR / script), "--help"],
|
||||
capture_output=True, text=True, timeout=120,
|
||||
encoding="utf-8", errors="replace")
|
||||
assert proc.returncode == 0, "%s --help exited %d: %s" % (script, proc.returncode, proc.stderr)
|
||||
assert "usage:" in proc.stdout.lower(), "%s printed no usage block" % script
|
||||
assert script in proc.stdout, "%s does not name itself in its usage line" % script
|
||||
@@ -390,8 +390,48 @@ def test_prober_rows_carry_a_measurement_and_a_note(hwprobe_report):
|
||||
assert row["state"] in ROUTE_STATES or row["state"].startswith("needs-"), row
|
||||
|
||||
|
||||
def test_prober_can_dump_raw_cpuid_registers(run_script):
|
||||
def test_prober_can_dump_raw_cpuid_registers(run_script, hwprobe_report):
|
||||
"""A requested CPUID dump must be non-empty -- where CPUID is reachable at all.
|
||||
|
||||
The probe's own report says which of those two situations this host is in, so the test reads it
|
||||
instead of assuming x64 Windows: on a non-x64 or non-Windows host the stub cannot run, and
|
||||
"nothing was reported" is the correct answer rather than a defect. A missing capability is a
|
||||
skip with the probe's own note, never a red test (tests/README.md, conftest.py).
|
||||
"""
|
||||
cpu = hwprobe_report["cpu"][0]
|
||||
if cpu["state"] != "available":
|
||||
pytest.skip("CPUID is not reachable on this host, so there is no register dump to check: %s"
|
||||
% cpu["note"])
|
||||
result = run_script("hwprobe.py", "--raw", "--json", timeout=120)
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["raw_cpuid"], "a raw dump was requested but nothing was reported"
|
||||
|
||||
|
||||
# --- pe_triage ---------------------------------------------------------------
|
||||
|
||||
def test_triage_reports_identity_and_leaves_the_sample_untouched(fixtures, run_script):
|
||||
"""The first script a task should run: it parses, classifies, and must not write."""
|
||||
sample = fixtures.pe64
|
||||
before = sample.data
|
||||
result = run_script("pe_triage.py", sample.path, "--json")
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["sha256"] == sample.sha256
|
||||
assert payload["identity"]["arch"] == "amd64"
|
||||
assert payload["identity"]["bitness"] == 64
|
||||
assert payload["identity"]["kind"] == "exe"
|
||||
assert sample.data == before, "triage modified the image it was reading"
|
||||
|
||||
|
||||
# --- pe_diff -----------------------------------------------------------------
|
||||
|
||||
def test_diff_of_the_same_build_reports_no_material_change(fixtures, run_script):
|
||||
"""Self-comparison is the control for every other diff: it must find nothing."""
|
||||
sample = fixtures.pe64
|
||||
result = run_script("pe_diff.py", sample.path, sample.path, "--json")
|
||||
assert result.returncode == 0, result.stderr
|
||||
payload = json.loads(result.stdout)
|
||||
assert payload["verdict"] == "identical-modulo-normalised-fields"
|
||||
assert payload["material_change"] is False
|
||||
assert payload["byte_diff"]["differ_bytes"] == 0
|
||||
|
||||
Reference in New Issue
Block a user