feat(types): spec-generated REST/RELAY TypedDicts + wired return types + GEN-FRESH gate

.coveragerc

@@ -9,6 +9,13 @@ omit =
     setup.py
     */examples/*
     */docs/*
+    # Spec-generated static type stubs (TypedDicts emitted from the canonical
+    # specs). Type-only — imported solely under TYPE_CHECKING, never executed at
+    # runtime, so they carry no testable statements; counting them would dilute
+    # coverage with ~1500 unreachable lines. Regenerated + freshness-gated by the
+    # GEN-FRESH gate, not by tests.
+    */rest/namespaces/*_types_generated.py
+    */relay/protocol_types_generated.py
 
 [report]
 # Regexes for lines to exclude from consideration

.gitignore

@@ -247,3 +247,6 @@ Anaconda3-2024.02-1-Linux-x86_64.sh
 chunks.json
 my_chunks
 *.swsearch
+
+# run-ci gate-scheduler scratch (repo-local; auto-cleaned)
+.sw-tmp/

pyproject.toml

@@ -182,6 +182,16 @@ select = ["E4", "E7", "E9", "F", "B", "S", "C4", "PERF", "SIM", "PTH", "RET", "R
 # call) and S607 (partial executable path: git/ssh resolved via PATH, the normal
 # idiom) are expected and safe across this file.
 "signalwire/signalwire/cli/dokku.py" = ["S603", "S607"]
+# REST namespace files import spec-generated types under TYPE_CHECKING and use
+# them ONLY inside quoted forward-refs — method return annotations
+# (`-> "RoomResponse"`) and the generic CRUD-base subscript bindings
+# (`class VideoRooms(CrudResource["ListRoomsResponse", "RoomResponse", ...])`).
+# Ruff's F401 cannot see usages inside a class-base subscript's string literals,
+# so it false-positives "unused import" and `--fix` would strip imports that ARE
+# used (breaking mypy). The bindings must stay quoted (a bare name in the base
+# subscript is a runtime NameError under TYPE_CHECKING). Exempt F401 here; mypy +
+# the SIGNATURES/DRIFT gate prove these imports are real and correct.
+"signalwire/signalwire/rest/namespaces/*.py" = ["F401"]
 
 [tool.ruff.lint.flake8-bugbear]
 # FastAPI's dependency-injection idiom puts Depends()/Query()/etc. in argument
@@ -202,16 +212,22 @@ extend-immutable-calls = [
 
 [tool.mypy]
 python_version = "3.10"
-files = ["signalwire/signalwire"]
+# Both the source tree AND the test suite are in scope — the generated REST tests are
+# strict-clean by construction, and the hand-written tests were fully annotated, so the
+# strict bar now covers tests too (a new untyped test fails the gate).
+files = ["signalwire/signalwire", "tests"]
 # Third-party packages without type stubs (nltk, sentence_transformers, the
 # search/relay stacks) are out of scope — we type our own code, not theirs.
 ignore_missing_imports = true
-# The bar: our own code is clean. Bodies of untyped defs are checked so real
-# type errors inside them surface (this is what found the mixin cluster).
-check_untyped_defs = true
-warn_unused_ignores = true
-# `# type: ignore` is reserved for genuinely-unfixable third-party-stub gaps and
-# must carry an error code + reason. warn_unused_ignores keeps them honest.
+# The bar: the whole source tree is mypy --strict clean (full annotations + all
+# the strict sub-flags). Achieved by typing handlers with REAL types (FastAPI
+# Response/dict, generated payload types where they exist) — never papered with
+# bare `Any`. `strict` subsumes check_untyped_defs / warn_unused_ignores / etc.
+strict = true
+# `# type: ignore` is reserved for genuinely-unfixable third-party-stub gaps (e.g.
+# flask/flask-limiter optional extras with no stubs) and the documented
+# _HostTyped/AgentBase TYPE_CHECKING-vs-runtime split; each must carry an error
+# code + reason. warn_unused_ignores (via strict) keeps them honest.
 show_error_codes = true
 
 # numpy (a search/relay-extra runtime dep we do NOT type — same class as nltk /

scripts/run-ci.sh

@@ -4,23 +4,27 @@
 # Same script invoked locally (`bash scripts/run-ci.sh`) AND by the
 # GitHub Actions workflow. No drift between local and CI behavior.
 #
-# Python is the reference SDK. Per-port "drift gate" doesn't apply —
-# python IS the spec. Instead we regenerate the python_signatures.json
-# oracle (in porting-sdk/) from the live Python source and verify the
-# regenerated file matches what's checked in. That way a Python-side
-# surface change can't silently invalidate every other port's audit.
+# Python is the reference SDK. Per-port "drift gate" doesn't apply — python IS the
+# spec. Instead we regenerate the python_signatures.json oracle (in porting-sdk/)
+# from the live Python source and verify the regenerated file matches what's checked
+# in, so a Python-side surface change can't silently invalidate every other port's audit.
 #
-# Gates (in order, fail-fast):
-#   1. python -m pytest tests/unit/       — language test runner
-#   2. signature regen                    — porting-sdk's enumerate_python_signatures.py
-#   3. drift gate                         — git-diff python_signatures.json (must be unchanged)
-#   4. no-cheat gate                      — porting-sdk audit_no_cheat_tests.py
-#   4b. rest-coverage gate                — porting-sdk rest_coverage.py over a REST-suite
-#                                           journal: every implemented route hit success+error
-#                                           (parity), accepted gaps allowlisted
-#   5. fmt gate                           — ruff format (local: apply; CI: --check)
-#   6. lint gate                          — ruff check, zero findings
-#   7. typecheck gate                     — mypy, zero findings
+# GATE SCHEDULING (porting-sdk/scripts/gate_scheduler.sh — CI_PERF S1 + S2):
+#   Gates run CONCURRENTLY up to a cap (SW_CI_JOBS, default nproc), scheduled by
+#   their DATA dependencies:
+#     * S2 concurrent wave: the pure-Python side-effect-free gates (NO-CHEAT,
+#       SPEC-PARITY, LINT, TYPECHECK, GEN-FRESH) overlap — they share no mutable state.
+#     * S1 fail-fast: heavy gates (TEST, REST-COVERAGE, FMT) are deferred behind the
+#       cheap wave, so a trivial cheap-gate failure surfaces in seconds; --fail-fast
+#       aborts the run before TEST starts.
+#   HARD ordering is data-dependency ONLY:
+#     * SIGNATURES regenerates python_signatures.json in porting-sdk; DRIFT git-diffs
+#       that file → deps=SIGNATURES (never diff before the regen writes).
+#   Per-gate PASS/FAIL + the FAILED_GATES tally preserved exactly; each gate's output
+#   captured + replayed atomically.
+#
+# Flags:
+#   --fail-fast   stop launching new gates at the first failure (local dev loop).
 
 set -u
 set -o pipefail
@@ -46,37 +50,9 @@ PORTING_SDK_DIR="$(resolve_porting_sdk)" || {
     exit 2
 }
 
-FAILED_GATES=""
-
-run_gate() {
-    local name="$1"; shift
-    local description="$1"; shift
-    local logfile
-    logfile="$(mktemp)"
-    "$@" >"$logfile" 2>&1
-    local rc=$?
-    if [ "$rc" -eq 0 ]; then
-        echo "[$name] $description ... PASS"
-        rm -f "$logfile"
-        return 0
-    fi
-    echo "[$name] $description ... FAIL: exit $rc"
-    # On failure show the LAST 400 lines (was 40) so pytest's per-test
-    # tracebacks — which print before the short-summary — actually reach the CI
-    # log. A blind `tail -40` on a multi-thousand-test run captured only the
-    # FAILED summary lines and discarded every traceback, making CI-only
-    # failures undiagnosable. 400 lines covers a realistic handful of failing
-    # tests; a fully green run prints nothing here regardless.
-    sed 's/^/    /' "$logfile" | tail -400
-    rm -f "$logfile"
-    FAILED_GATES="$FAILED_GATES $name"
-    return $rc
-}
+# shellcheck source=/dev/null
+source "$PORTING_SDK_DIR/scripts/gate_scheduler.sh"
 
-# Ask the OS for a free TCP port (bind :0, read it back, release). Used by the
-# REST-COVERAGE gate so a stale mock_signalwire squatting a fixed port can't
-# silently steal the gate's traffic (a previously hardcoded 8951 did exactly
-# that). Prints the port on stdout; nonzero exit on failure (caller fails loud).
 pick_free_port() {
     python3 - <<'PY'
 import socket
@@ -91,73 +67,42 @@ cd "$PORT_ROOT"
 
 echo "==> running CI gates for $PORT_NAME (porting-sdk at $PORTING_SDK_DIR)"
 
-# Record the resolved web-stack versions up front. CI-only test failures in the
-# web layer have been impossible to diagnose because the runner's fresh
-# dependency resolution differs from local envs; logging it makes that visible.
+# Record the resolved web-stack versions up front (CI-only web-layer failures have
+# been impossible to diagnose without the runner's actual resolution).
 echo "==> environment: $(python3 --version 2>&1)"
 python3 -m pip freeze 2>/dev/null \
     | grep -iE '^(fastapi|starlette|pydantic|pydantic-core|anyio|uvicorn|httpx|requests)==' \
     | sed 's/^/    /' || true
 
-# Gate 1: pytest unit suite
-run_gate "TEST" "python -m pytest tests/unit/" \
-    python -m pytest tests/unit/
-
-# Gate 2: regenerate the python reference oracle into porting-sdk
-run_gate "SIGNATURES" "regenerate python_signatures.json (reference oracle)" \
-    python3 "$PORTING_SDK_DIR/scripts/enumerate_python_signatures.py" \
-        --signalwire-python "$PORT_ROOT/signalwire" \
-        --out "$PORTING_SDK_DIR/python_signatures.json"
+# FMT — ruff format. LOCAL applies; CI --check.
+fmt_gate() {
+    if [ -n "${CI:-}" ]; then
+        python3 -m ruff format --check "$PORT_ROOT/signalwire"
+    else
+        python3 -m ruff format "$PORT_ROOT/signalwire" >/dev/null
+        if ! (cd "$PORT_ROOT" && git diff --quiet 2>/dev/null); then
+            echo "    (FMT auto-applied formatting to your working tree — review & stage)"
+        fi
+        python3 -m ruff format --check "$PORT_ROOT/signalwire"
+    fi
+}
 
-# Gate 3: oracle-drift gate — the regenerated file must match what's
-# committed in porting-sdk. Drift here means a Python-side surface
-# change wasn't propagated to the audit oracle, which would silently
-# invalidate every port's drift report.
-run_gate "DRIFT" "python_signatures.json unchanged after regen" \
-    bash -c "cd '$PORTING_SDK_DIR' && git diff --quiet -- python_signatures.json"
-
-# Gate 4: no-cheat
-run_gate "NO-CHEAT" "audit_no_cheat_tests" \
-    python3 "$PORTING_SDK_DIR/scripts/audit_no_cheat_tests.py" --root "$PORT_ROOT"
-
-# Gate 4b: REST-COVERAGE — every canonical REST route the SDK implements must be
-# exercised with BOTH a success (2xx) AND an error (4xx/5xx) response, on the
-# correct on-the-wire path (parity). Measured by replaying the mock journal of a
-# REST-suite run through porting-sdk's rest_coverage checker. Accepted gaps —
-# routes with no SDK method, malformed canonical routes, mock-router collisions —
-# are allowlisted: the shared baseline (porting-sdk/REST_COVERAGE_BASELINE.md) for
-# universal gaps + this port's REST_COVERAGE_GAPS.md for its own sdk-gaps. A
-# stale allowlist entry (a route now actually covered) fails the gate.
-#
-# Self-contained: spins up its own mock on an ephemeral port, runs the rest/ suite
-# serially against it (MOCK_SIGNALWIRE_PORT so all traffic lands in one journal),
-# then checks that journal. Same shape on every port.
-#
-# The mock + checker are run as `-m mock_signalwire.<mod>`; that package lives at
-# porting-sdk/test_harness/mock_signalwire/ and is NOT pip-installed in CI (the
-# runner only checks porting-sdk out). Put the package parent on PYTHONPATH so the
-# module resolves regardless of install state — local and CI, every port.
+# REST-COVERAGE — every implemented REST route covered success+error. Self-
+# contained: spins its own mock on an ephemeral port, runs the rest/ suite serially,
+# then checks the journal.
 rest_coverage_gate() {
     local port
     port="$(pick_free_port)" || {
         echo "FATAL: could not acquire a free port for mock_signalwire" >&2
         return 1
     }
-    # mock_signalwire is pip-installed (-e) in CI from porting-sdk/test_harness/.
-    # Locally it may be discovered via adjacency instead; put the package parent on
-    # PYTHONPATH as a belt-and-suspenders fallback so `-m mock_signalwire.*` imports
-    # regardless of install state. (rest_coverage.py must exist in the checked-out
-    # porting-sdk — it ships in the same change that adds this gate.)
     local mock_pkg_parent="$PORTING_SDK_DIR/test_harness/mock_signalwire"
     export PYTHONPATH="$mock_pkg_parent${PYTHONPATH:+:$PYTHONPATH}"
     python3 -m mock_signalwire --host 127.0.0.1 --port "$port" --log-level error \
         >/tmp/rest_cov_mock.$$.log 2>&1 &
     local mock_pid=$!
     # shellcheck disable=SC2064
     trap "kill $mock_pid 2>/dev/null" RETURN
-    # Wait for the control plane to answer. Fail LOUD if the mock never becomes
-    # healthy (or dies during startup) instead of proceeding silently into a
-    # journal-reset that 404s/connection-refuses with an opaque traceback.
     local i healthy=0
     for i in $(seq 1 30); do
         if ! kill -0 "$mock_pid" 2>/dev/null; then
@@ -185,59 +130,51 @@ rest_coverage_gate() {
         --allowlist "$PORT_ROOT/REST_COVERAGE_GAPS.md" \
         --gap-baseline "$PORTING_SDK_DIR/REST_COVERAGE_GAP_BASELINE.md"
 }
-run_gate "REST-COVERAGE" "every implemented REST route covered success+error (parity + allowlist)" \
-    rest_coverage_gate
-
-# Gate 4c: SPEC-PARITY — the routes the SDK actually IMPLEMENTS must equal the
-# canonical spec route set, modulo a checked-in not-implemented gaps file. This
-# is the spec-first guard: REST-COVERAGE only proves tested routes match the
-# spec, so a route the SDK implements but the spec doesn't define (or vice
-# versa) would slip past it. The route registry is built by driving the live
-# client through a recording HttpClient (not an AST scrape, not the test
-# journal), so it sees every implemented route whether or not it's tested.
-#   * B−A (SDK route absent from the spec) → fail (new API must enter the spec)
-#   * A−B (canonical route not implemented) → fail unless in SPEC_IMPLEMENTATION_GAPS.md
-run_gate "SPEC-PARITY" "implemented routes == canonical spec (modulo SPEC_IMPLEMENTATION_GAPS.md)" \
-    python3 "$PORTING_SDK_DIR/scripts/diff_spec_implementation.py" \
+
+# ---- register gates ----------------------------------------------------------
+sched_init "$@"
+
+sched_gate TEST defer=1 desc="python -m pytest tests/unit/" \
+    -- python -m pytest tests/unit/
+
+# SIGNATURES regenerates the porting-sdk oracle → DRIFT git-diffs it. deps=SIGNATURES.
+sched_gate SIGNATURES desc="regenerate python_signatures.json (reference oracle)" \
+    -- python3 "$PORTING_SDK_DIR/scripts/enumerate_python_signatures.py" \
+        --signalwire-python "$PORT_ROOT/signalwire" \
+        --out "$PORTING_SDK_DIR/python_signatures.json"
+
+sched_gate DRIFT deps=SIGNATURES desc="python_signatures.json unchanged after regen" \
+    -- bash -c "cd '$PORTING_SDK_DIR' && git diff --quiet -- python_signatures.json"
+
+sched_gate NO-CHEAT desc="audit_no_cheat_tests" \
+    -- python3 "$PORTING_SDK_DIR/scripts/audit_no_cheat_tests.py" --root "$PORT_ROOT"
+
+sched_gate REST-COVERAGE defer=1 desc="every implemented REST route covered success+error (parity + allowlist)" \
+    --fn rest_coverage_gate
+
+sched_gate SPEC-PARITY defer=1 desc="implemented routes == canonical spec (modulo SPEC_IMPLEMENTATION_GAPS.md)" \
+    -- python3 "$PORTING_SDK_DIR/scripts/diff_spec_implementation.py" \
         --sdk "$PORT_ROOT/signalwire/signalwire" \
         --gaps "$PORTING_SDK_DIR/SPEC_IMPLEMENTATION_GAPS.md"
 
-# Gate 5: FMT — ruff format. Config in pyproject.toml [tool.ruff]. Source-style
-# only (no public-API change → the SIGNATURES/DRIFT oracle is unaffected).
-#   * LOCAL ($CI unset)  → `ruff format` reformats the tree in place.
-#   * CI ($CI=true)      → `ruff format --check` (read-only): fails on unformatted.
-fmt_gate() {
-    if [ -n "${CI:-}" ]; then
-        python3 -m ruff format --check "$PORT_ROOT/signalwire"
-    else
-        python3 -m ruff format "$PORT_ROOT/signalwire" >/dev/null
-        if ! (cd "$PORT_ROOT" && git diff --quiet 2>/dev/null); then
-            echo "    (FMT auto-applied formatting to your working tree — review & stage)"
-        fi
-        python3 -m ruff format --check "$PORT_ROOT/signalwire"
-    fi
-}
-run_gate "FMT" "ruff format (local: apply; CI: --check)" fmt_gate
-
-# Gate 6: LINT — ruff check, zero findings. Burned to zero (the DISABLE list is
-# none; the few intentional lazy-import / re-export sites carry per-line # noqa
-# with a rationale). Mirrors the go golangci / ruby rubocop blocking-lint gate.
-run_gate "LINT" "ruff check zero findings" \
-    python3 -m ruff check "$PORT_ROOT/signalwire"
-
-# Gate 7: TYPECHECK — mypy, zero findings. Config in pyproject.toml [tool.mypy]
-# (files=signalwire/signalwire, ignore_missing_imports, check_untyped_defs). Burned
-# to zero across all 176 source files. `# type: ignore` is reserved for genuine
-# third-party-stub gaps / optional-dependency import shims, each with an error code
-# and rationale (warn_unused_ignores keeps them honest). Source-type only — mypy is
-# dev-time and adds no runtime validation, so wire payloads stay forward-compatible.
-run_gate "TYPECHECK" "mypy zero findings" \
-    python3 -m mypy --config-file "$PORT_ROOT/pyproject.toml"
-
-if [ -z "$FAILED_GATES" ]; then
+sched_gate FMT defer=1 desc="ruff format (local: apply; CI: --check)" \
+    --fn fmt_gate
+
+sched_gate LINT desc="ruff check zero findings" \
+    -- python3 -m ruff check "$PORT_ROOT/signalwire"
+
+sched_gate TYPECHECK desc="mypy zero findings" \
+    -- python3 -m mypy --config-file "$PORT_ROOT/pyproject.toml"
+
+sched_gate GEN-FRESH desc="generated REST/RELAY types reproduce from specs" \
+    -- python3 "$PORTING_SDK_DIR/scripts/generate_python_rest_types.py" \
+        --signalwire-python "$PORT_ROOT/signalwire" --check
+
+sched_run
+rc=$?
+if [ "$rc" -eq 0 ]; then
     echo "==> CI PASS"
-    exit 0
 else
     echo "==> CI FAIL (gates:$FAILED_GATES )"
-    exit 1
 fi
+exit "$rc"