Fix the budget gate and add CI, lint and coverage guards

- check_budget.py escalated a hard index-row violation to 'warn' on both
  branches, so INDEX_ROW_HARD could never fail anything. It fails now, and
  tests/unit/test_budget_measurement.py pins both thresholds.
- The corpus note counted scripts/__pycache__/*.pyc, so the same commit
  reported 27 scripts after the suite had run and 16 on a clean checkout.
- TOKENS_PER_CHAR is re-derived from a measurement (26,795 chars -> 6,065
  tokens = 4.418); the comment's old figure and its arithmetic were both
  wrong for this file.
- check_repo.py's yaml sentinel carries the type ignore, so mypy is clean
  with PyYAML installed as well as without it.
- The CPUID test now skips with the probe's own note on a host without
  CPUID instead of failing, and pytest.ini points at tests/README.md, which
  exists, instead of an EXTENSION file that never did.

Tooling: .flake8 (a measured 120-column limit), .gitattributes, .gitignore
and .github/workflows/ci.yml (the five gates, the suite on 3.11-3.14 and a
lint job), which is what CONTRIBUTING.md already claimed CI runs. The
pull-request template it refers to now exists.

Content: pitfalls P26-P34 -- the analysis-level failures the catalogue was
missing (sampling read as observation, a self-built parser trusted, a
verification that could not have failed, a shell-mangled command line) --
registered in the symptom index and its routing mirror. FINDINGS F11-F15
record the defects above.

Tests: the coverage guard derives its subject from scripts/ rather than a
hand-written list, pe_triage.py and pe_diff.py gain their first tests, and
plan.py / preflight.py are declared uncovered with reasons.
This commit is contained in:
root
2026-10-24 14:55:02 +08:00
parent 25f2286cb6
commit 63b2aca8d8
19 changed files with 812 additions and 14 deletions
+80
View File
@@ -0,0 +1,80 @@
"""Coverage of the shipped scripts, derived from the directory rather than a hand-written list.
Borrowed mechanism: the sibling apk-reverse repository derives its CLI contract cases from
skills/*/scripts/*.py and from each script's own --help output, so a new script is covered the day
it lands instead of the day someone remembers to add it. What is *not* borrowed is that repository's
single 0-4 exit-code table: this kit publishes a per-script vocabulary (pepatch.py uses 3 for "not a
parseable PE", where the table means "the environment lacks a capability"), and a test that rewrote
those meanings would assert a policy this repository deliberately does not have.
The guard here is therefore narrower, and it is about *coverage*, not exit codes:
* every shipped .py is either in conftest.SCRIPT_FILES (some layer of this suite exercises it) or
named in DECLARED_UNCOVERED together with the reason it is not;
* every declared gap carries a real sentence, because "TODO" is how a gap becomes permanent;
* every shipped script -- covered or declared -- answers --help with its usage block, which is the
cheapest possible exercise of its argparse table and its import path.
A new script fails this file until one of those lists changes. That is the point: the alternative
is a suite that looks complete while silently ignoring half the kit.
"""
import subprocess
import sys
import pytest
from conftest import SCRIPTS_DIR, SCRIPT_FILES
pytestmark = pytest.mark.cli
# Scripts no layer exercises yet. Each entry is a recorded gap, not an exemption: the reason says
# what would be needed to close it, so the next pass can decide rather than rediscover.
DECLARED_UNCOVERED = {
"plan.py":
"composes capabilities.py and doctor.py into a route from a stated intent. Both halves have "
"their own tests; the composition reads live host facts, so a fixture that pinned its output "
"would pin this machine's capability set instead of the mapping. Closing it needs a stubbed "
"probe, and that stub does not exist yet.",
"preflight.py":
"answers session readiness from live host facts (free space, route prerequisites) plus one "
"sample. The sample half could be pinned to a generated PE; the host half has no generator, "
"and asserting on the runner's environment would encode this machine as an expectation.",
}
def shipped_scripts():
"""Every runnable python script the skill ships, sorted."""
return sorted(p.name for p in SCRIPTS_DIR.glob("*.py"))
def test_every_shipped_script_is_exercised_or_declared():
"""A script nobody tests must at least be a script somebody decided not to test."""
shipped = set(shipped_scripts())
assert shipped, "no scripts found under %s" % SCRIPTS_DIR
silently_uncovered = shipped - set(SCRIPT_FILES) - set(DECLARED_UNCOVERED)
assert not silently_uncovered, (
"these scripts are shipped but neither exercised nor declared uncovered: %s. Add a test and "
"register the script in conftest.SCRIPT_FILES, or add it to DECLARED_UNCOVERED with the "
"reason." % sorted(silently_uncovered))
def test_declared_entries_are_neither_stale_nor_hollow():
"""A gap that was closed must leave the list; a gap with no reason is a hidden one."""
both = set(SCRIPT_FILES) & set(DECLARED_UNCOVERED)
assert not both, ("these scripts are in SCRIPT_FILES and also declared uncovered: %s -- the "
"declaration is stale" % sorted(both))
for name, reason in DECLARED_UNCOVERED.items():
assert (SCRIPTS_DIR / name).is_file(), "%s is declared uncovered but does not exist" % name
assert len(reason.split()) >= 12, (
"%s: the reason must say what would close the gap, not just mark it" % name)
@pytest.mark.parametrize("script", shipped_scripts())
def test_every_shipped_script_answers_help_with_usage(script):
"""The usage block is the script's published contract; it must render on every host."""
proc = subprocess.run([sys.executable, str(SCRIPTS_DIR / script), "--help"],
capture_output=True, text=True, timeout=120,
encoding="utf-8", errors="replace")
assert proc.returncode == 0, "%s --help exited %d: %s" % (script, proc.returncode, proc.stderr)
assert "usage:" in proc.stdout.lower(), "%s printed no usage block" % script
assert script in proc.stdout, "%s does not name itself in its usage line" % script
+41 -1
View File
@@ -390,8 +390,48 @@ def test_prober_rows_carry_a_measurement_and_a_note(hwprobe_report):
assert row["state"] in ROUTE_STATES or row["state"].startswith("needs-"), row
def test_prober_can_dump_raw_cpuid_registers(run_script):
def test_prober_can_dump_raw_cpuid_registers(run_script, hwprobe_report):
"""A requested CPUID dump must be non-empty -- where CPUID is reachable at all.
The probe's own report says which of those two situations this host is in, so the test reads it
instead of assuming x64 Windows: on a non-x64 or non-Windows host the stub cannot run, and
"nothing was reported" is the correct answer rather than a defect. A missing capability is a
skip with the probe's own note, never a red test (tests/README.md, conftest.py).
"""
cpu = hwprobe_report["cpu"][0]
if cpu["state"] != "available":
pytest.skip("CPUID is not reachable on this host, so there is no register dump to check: %s"
% cpu["note"])
result = run_script("hwprobe.py", "--raw", "--json", timeout=120)
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout)
assert payload["raw_cpuid"], "a raw dump was requested but nothing was reported"
# --- pe_triage ---------------------------------------------------------------
def test_triage_reports_identity_and_leaves_the_sample_untouched(fixtures, run_script):
"""The first script a task should run: it parses, classifies, and must not write."""
sample = fixtures.pe64
before = sample.data
result = run_script("pe_triage.py", sample.path, "--json")
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout)
assert payload["sha256"] == sample.sha256
assert payload["identity"]["arch"] == "amd64"
assert payload["identity"]["bitness"] == 64
assert payload["identity"]["kind"] == "exe"
assert sample.data == before, "triage modified the image it was reading"
# --- pe_diff -----------------------------------------------------------------
def test_diff_of_the_same_build_reports_no_material_change(fixtures, run_script):
"""Self-comparison is the control for every other diff: it must find nothing."""
sample = fixtures.pe64
result = run_script("pe_diff.py", sample.path, sample.path, "--json")
assert result.returncode == 0, result.stderr
payload = json.loads(result.stdout)
assert payload["verdict"] == "identical-modulo-normalised-fields"
assert payload["material_change"] is False
assert payload["byte_diff"]["differ_bytes"] == 0