Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 22 additions & 1 deletion .github/rulesets/main.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,27 @@
"rules": [
{ "type": "pull_request" },
{ "type": "non_fast_forward" },
{ "type": "deletion" }
{ "type": "deletion" },
{
"type": "required_status_checks",
"parameters": {
"required_status_checks": [
{ "context": "quality" },
{ "context": "tests" },
{ "context": "security" },
{ "context": "docs-privacy" },
{ "context": "Analyze (python)" }
],
"strict_required_status_checks_policy": true
}
},
{
"type": "code_scanning",
"parameters": {
"code_scanning_tools": [
{ "tool": "CodeQL", "security_alerts_threshold": "high_or_higher", "alerts_threshold": "errors" }
]
}
}
]
}
4 changes: 2 additions & 2 deletions ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ gate enforces that this block matches the registries.
| `scripts/check_docs_private_markers.py` | `L1`, `L2`, `L3`, `L4` | `test_check_docs_private_markers.py` |
| `scripts/check_floor_coverage.py` | `F1`, `F2`, `F3`, `F4` | `test_check_floor_coverage.py` |
| `scripts/check_job_deps.py` | `J1`, `J2`, `J3` | `test_check_job_deps.py` |
| `scripts/check_required_checks.py` | `RC1`, `RC2` | `test_check_required_checks.py` |
| `scripts/check_required_checks.py` | `RC1`, `RC2`, `RC3` | `test_check_required_checks.py` |
| `scripts/discover_floors.py` | (no codes) | `test_discover_floors.py` |
| `scripts/gen_architecture.py` | (no codes) | `test_gen_architecture.py` |
| `scripts/read_ci_verdict.py` | (no codes) | `test_read_ci_verdict.py` |
Expand All @@ -142,6 +142,6 @@ gate enforces that this block matches the registries.
| `skills/project-pilot/modules/steering-docs-editing/scripts/check_steering_doc.py` | `A1`, `A2`, `A3`, `B-impact`, `D1a`, `D1b`, `D2`, `D3`, `G1`, `H1`, `R1a`, `R1b`, `R2`, `R3`, `R4`, `R5` | `test_check_steering_doc.py` |
| `skills/project-pilot/modules/steering-docs-editing/scripts/reslice_journal.py` | (no codes) | `test_reslice_journal.py` |

_19 tool(s), 74 declared code(s) - regenerated by `scripts/gen_architecture.py` from the `CODES`/`FLOOR` registries._
_19 tool(s), 75 declared code(s) - regenerated by `scripts/gen_architecture.py` from the `CODES`/`FLOOR` registries._

<!-- ARCH:INVENTORY:END -->
61 changes: 52 additions & 9 deletions scripts/check_required_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
read_ci_verdict consumes .github/required-checks.json, the canonical gate policy
(family G7). The doc-drift `gate` kind already proves the *prose* matches that
JSON; nothing proved the JSON matches the *repo*. This meta-check closes that
half (the second part of the required-checks work), on two decidable rules:
half (the second part of the required-checks work), on three decidable rules:

RC1 - a `decisional` name has no scripts/gates/<name>.sh runner. The runners
are the single source of what each gate runs (CONTRIBUTING.md), and each
decisional gate's workflow calls exactly `bash scripts/gates/<name>.sh`,
so a present runner is the stdlib-exact witness that the gate is real and
runnable -- no YAML parser needed (the repo ships none; parsing job names
would be the fragile heuristic METHOD §6 rejects). This is *why*
would be the fragile heuristic METHOD section 6 rejects). This is *why*
"Analyze (python)" cannot be decisional: it has no runner (it is gated by
a code-scanning ruleset, not a status check), so
listing it under `decisional` fires RC1.
Expand All @@ -20,28 +20,39 @@
on a check the branch protection does not even require; this ties the two
name sets so they cannot drift into disjoint sets.

RC3 - the branch ruleset's required_status_checks contexts (.github/rulesets/
main.json) do not match `required` as a set. `required` is the canonical
list; the ruleset payload is the native enforcement posted verbatim by
setup-layer-b.sh (no generation step). Hard-coding the list there mirrors
`required`, so RC3 makes that mirror decidable: any name required-but-not-
enforced, or enforced-but-not-required, reddens instead of drifting in
silence (the unguarded-mirror channel METHOD section 8, precedent C8).

The one channel RC1 leaves open -- a job renamed in YAML without renaming its
runner -- is closed by the live arm: read_ci_verdict runs on a real PR every
close, where a divergent name surfaces that decisional check as MISSING.

The core (`confront`) is pure and offline-testable; the thin shell collects the
runner names from the filesystem and the policy via load_policy.
runner names and the ruleset contexts from the filesystem and the policy via
load_policy.
"""

from __future__ import annotations

import json
import sys
from dataclasses import dataclass
from pathlib import Path

from read_ci_verdict import PolicyError, load_policy

# Floor-coverage claim (METHOD section 2 / v0.41): two rule codes, one floor.
CODES = frozenset({"RC1", "RC2"})
# Floor-coverage claim (METHOD section 2 / v0.41): three rule codes, one floor.
CODES = frozenset({"RC1", "RC2", "RC3"})
FLOOR = "test_check_required_checks.py"

_REPO_ROOT = Path(__file__).resolve().parent.parent
_REQUIRED_CHECKS_PATH = _REPO_ROOT / ".github" / "required-checks.json"
_RULESET_PATH = _REPO_ROOT / ".github" / "rulesets" / "main.json"
_GATES_DIR = _REPO_ROOT / "scripts" / "gates"


Expand All @@ -56,15 +67,22 @@ def confront(
decisional: tuple[str, ...],
required: tuple[str, ...],
runner_names: frozenset[str],
ruleset_contexts: frozenset[str],
) -> list[Finding]:
"""Policy defects, in `decisional` order. Pure: all inputs are passed in, so
the floor exercises the rules offline without touching the filesystem."""
"""Policy defects, in `decisional` order then RC3 (sorted). Pure: all inputs
are passed in, so the floor exercises the rules offline without touching the
filesystem."""
findings: list[Finding] = []
for name in decisional:
if name not in runner_names:
findings.append(Finding("RC1", name, f"no scripts/gates/{name}.sh runner"))
if name not in required:
findings.append(Finding("RC2", name, "decisional but not in required"))
required_set = frozenset(required)
for name in sorted(required_set - ruleset_contexts):
findings.append(Finding("RC3", name, "required but not in ruleset required_status_checks"))
for name in sorted(ruleset_contexts - required_set):
findings.append(Finding("RC3", name, "in ruleset required_status_checks but not required"))
return findings


Expand All @@ -73,19 +91,44 @@ def collect_runner_names(gates_dir: Path) -> frozenset[str]:
return frozenset(path.stem for path in gates_dir.glob("*.sh"))


def collect_ruleset_contexts(ruleset_path: Path) -> frozenset[str]:
"""The status-check contexts the branch ruleset requires (main.json). Single
source of the native-enforcement list, confronted against the policy's
`required` by RC3."""
data = json.loads(ruleset_path.read_text(encoding="utf-8"))
contexts: set[str] = set()
for rule in data.get("rules", []):
if rule.get("type") == "required_status_checks":
checks = rule.get("parameters", {}).get("required_status_checks", [])
for check in checks:
context = check.get("context")
if context is not None:
contexts.add(context)
return frozenset(contexts)


def main(argv: list[str] | None = None) -> int:
try:
policy = load_policy(_REQUIRED_CHECKS_PATH)
except PolicyError as exc:
print(f"error: {exc}", file=sys.stderr)
return 2
try:
contexts = collect_ruleset_contexts(_RULESET_PATH)
except (OSError, json.JSONDecodeError) as exc:
print(f"error: cannot read ruleset {_RULESET_PATH}: {exc}", file=sys.stderr)
return 2
names = collect_runner_names(_GATES_DIR)
findings = confront(policy.decisional, policy.required, names)
findings = confront(policy.decisional, policy.required, names, contexts)

print(f"Gate policy : decisional {sorted(policy.decisional)} vs runners {sorted(names)}")
print(f" required {sorted(policy.required)}")
print(f" ruleset {sorted(contexts)}")
if not findings:
print("\nRESULT : PASS (exit 0) - every decisional gate has a runner and is required")
print(
"\nRESULT : PASS (exit 0) - decisional gates runnable + required; "
"ruleset matches required"
)
return 0
for f in findings:
print(f" [{f.code}] {f.name!r}: {f.detail}")
Expand Down
42 changes: 30 additions & 12 deletions scripts/test_check_required_checks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,10 @@
"""Red floor for check_required_checks -- the deterministic core.

Proves the confront rules offline (no filesystem, no network): RC1 (a decisional
name with no runner) and RC2 (a decisional name not in required). Mutating
confront turns one of these red. Runs standalone (exit 0/1) and under pytest.
name with no runner), RC2 (a decisional name not in required), and RC3 (the
ruleset required_status_checks contexts do not match `required` as a set, either
direction). Mutating confront turns one of these red. Runs standalone (exit 0/1)
and under pytest.
"""

from __future__ import annotations
Expand All @@ -16,41 +18,57 @@
RUNNERS = frozenset({"quality", "tests", "security"})
REQUIRED = ("quality", "tests", "security", "Analyze (python)")
DECISIONAL = ("quality", "tests", "security")
RULESET = frozenset(REQUIRED) # a conforming ruleset mirrors `required` exactly


def _codes(findings):
return [f.code for f in findings]


def test_conforming_policy_has_no_finding():
assert confront(DECISIONAL, REQUIRED, RUNNERS) == []
assert confront(DECISIONAL, REQUIRED, RUNNERS, RULESET) == []


def test_decisional_without_runner_is_RC1():
assert _codes(confront(("quality", "ghost"), REQUIRED + ("ghost",), RUNNERS)) == ["RC1"]
required = REQUIRED + ("ghost",)
assert _codes(confront(("quality", "ghost"), required, RUNNERS, frozenset(required))) == ["RC1"]


def test_decisional_not_in_required_is_RC2():
# 'tests' has a runner but is (here) not listed in required -> RC2 only.
assert _codes(confront(("tests",), ("quality", "security"), RUNNERS)) == ["RC2"]
required = ("quality", "security")
assert _codes(confront(("tests",), required, RUNNERS, frozenset(required))) == ["RC2"]


def test_codeql_as_decisional_is_RC1_only():
# The required-check boundary, enforced structurally: "Analyze (python)" IS required but
# has no runner, so making it decisional fires RC1 (not RC2) -- exactly why
# it must stay out of `decisional`.
findings = confront(("Analyze (python)",), REQUIRED, RUNNERS)
# "Analyze (python)" IS required but has no runner, so making it decisional
# fires RC1 (not RC2) -- exactly why it must stay out of `decisional`.
findings = confront(("Analyze (python)",), REQUIRED, RUNNERS, RULESET)
assert _codes(findings) == ["RC1"]


def test_findings_follow_decisional_order():
codes = _codes(confront(("ghost", "tests"), ("quality",), RUNNERS))
required = ("quality",)
codes = _codes(confront(("ghost", "tests"), required, RUNNERS, frozenset(required)))
# ghost: no runner (RC1) + not required (RC2); tests: not required (RC2).
assert codes == ["RC1", "RC2", "RC2"]


def test_declared_vocabulary_is_RC1_and_RC2():
assert CODES == frozenset({"RC1", "RC2"})
def test_required_missing_from_ruleset_is_RC3():
# a name in `required` the ruleset payload omits -> under-enforcement.
required = REQUIRED + ("docs-privacy",)
ruleset = frozenset(REQUIRED) # omits docs-privacy
assert _codes(confront(DECISIONAL, required, RUNNERS, ruleset)) == ["RC3"]


def test_ruleset_extra_context_is_RC3():
# a context the ruleset enforces that `required` no longer lists -> stale.
ruleset = frozenset(REQUIRED) | {"stale"}
assert _codes(confront(DECISIONAL, REQUIRED, RUNNERS, ruleset)) == ["RC3"]


def test_declared_vocabulary_is_RC1_RC2_RC3():
assert CODES == frozenset({"RC1", "RC2", "RC3"})


def _run() -> int:
Expand Down
Loading