diff --git a/.github/benchmark/run_scenarios.py b/.github/benchmark/run_scenarios.py new file mode 100644 index 0000000..a4b57d8 --- /dev/null +++ b/.github/benchmark/run_scenarios.py @@ -0,0 +1,252 @@ +"""Run the behavioral scenario fixtures (tests/SCENARIOS.md, encoded in scenarios.json) +through an OpenAI-compatible endpoint and evaluate each output against its declarative +checks. Emits a markdown report for a PR comment and a JSON result with a pass flag. + +Check types: + forbid_regex — none of the patterns may match the output (case-insensitive) + require_regex — every pattern must match the output + verdict_in — the ai-check report's VERDICT field must equal one of the values + fraction_in — the AI-EDITED FRACTION field must equal one of the values + score_min/max — the OVERALL SCORE field must be >= / <= the value + length_ratio — output words / input words must fall inside [min, max] +""" + +import argparse +import json +import os +import re +import sys +import time + +from openai import OpenAI + +MARKER = "" + +SYSTEM = ( + "You have the following skill loaded. Apply it exactly as written.\n\n" + "\n{skill}\n" +) + + +def call(client, models, system, user): + """Try each model in order; within a model, retry transient errors with backoff. + A later list entry is the fallback when an earlier model's quota is exhausted + (e.g. the non-lite tier's small requests-per-day allowance).""" + last_err = None + for mi, model in enumerate(models): + attempts, waits = (3, (20, 40)) if mi < len(models) - 1 else (6, (20, 40, 80, 160, 300)) + for attempt in range(attempts): + try: + resp = client.chat.completions.create( + model=model, + max_tokens=1800, + temperature=0.4, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + ) + out = (resp.choices[0].message.content or "").strip() + if out: + return out, getattr(resp, "model", None) or model + raise RuntimeError("empty completion") + except Exception as e: # noqa: BLE001 - retry any transport/server hiccup + last_err = e + if attempt == attempts - 1: + break + print(f" [{model}] attempt {attempt + 1} failed ({e}); " + f"retrying in {waits[attempt]}s", file=sys.stderr) + time.sleep(waits[attempt]) + if mi < len(models) - 1: + print(f" [{model}] exhausted; falling back to {models[mi + 1]}", file=sys.stderr) + raise last_err + + +def report_field(text, name): + # Tolerate "FIELD: value", "**FIELD:** value", "FIELD | value" (table rows), etc. + m = re.search(rf"{re.escape(name)}\s*[:*\]|_-]*\s*([A-Za-z][A-Za-z /-]*)", text, re.I) + if not m: + return None + return re.sub(r"[^A-Za-z -]", "", m.group(1)).strip() + + +def report_score(text): + m = re.search(r"OVERALL SCORE\s*[:*\]|_-]*\s*(\d+)", text, re.I) + return int(m.group(1)) if m else None + + +def evaluate(scenario, out): + failures = [] + for check in scenario["checks"]: + ctype = check["type"] + if ctype == "forbid_regex": + for pat in check["patterns"]: + m = re.search(pat, out, re.I) + if m: + failures.append(f"forbidden `{pat}` matched: `{m.group(0)[:60]}`") + elif ctype == "require_regex": + for pat in check["patterns"]: + if not re.search(pat, out, re.I): + failures.append(f"required `{pat}` not found ({check.get('label', '')})") + elif ctype == "verdict_in": + v = report_field(out, "VERDICT") + if v is None or v.lower() not in [x.lower() for x in check["values"]]: + failures.append(f"VERDICT `{v}` not in {check['values']}") + elif ctype == "fraction_in": + v = report_field(out, "AI-EDITED FRACTION") + if v is None or v.lower() not in [x.lower() for x in check["values"]]: + failures.append(f"AI-EDITED FRACTION `{v}` not in {check['values']}") + elif ctype == "score_min": + s = report_score(out) + if s is None or s < check["value"]: + failures.append(f"OVERALL SCORE `{s}` below required {check['value']}") + elif ctype == "score_max": + s = report_score(out) + if s is None or s > check["value"]: + failures.append(f"OVERALL SCORE `{s}` above allowed {check['value']}") + elif ctype == "length_ratio": + ratio = len(out.split()) / max(1, len(scenario["input"].split())) + if not (check["min"] <= ratio <= check["max"]): + failures.append( + f"length ratio {ratio:.2f} outside [{check['min']}, {check['max']}]") + else: + failures.append(f"unknown check type `{ctype}`") + return failures + + +def block(scenario, out, failures, cap): + text = out if len(out) <= cap else out[:cap] + " … _(truncated; full text in the run artifact)_" + status = "✅" if not failures else "❌" + lines = [ + f"
{status} scenario {scenario['id']} · {scenario['name']} " + f"(`{scenario['skill']}`)", + "", + ] + if failures: + lines += ["**Failed checks:**", ""] + [f"- {f}" for f in failures] + [""] + lines += ["**Output:**", "", "> " + text.replace("\n", "\n> "), "", "
"] + return "\n".join(lines) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--scenarios", required=True) + p.add_argument("--humanize-skill", required=True) + p.add_argument("--ai-check-skill", required=True) + p.add_argument("--model", required=True) + p.add_argument("--strong-model", default=None, + help="Model for ai-check scenarios (judgment-heavy; the lite tier " + "measurably can't do them). Falls back to --model when its " + "daily quota is exhausted.") + p.add_argument("--base-url", required=True) + p.add_argument("--report", required=True) + p.add_argument("--json-out", required=True) + p.add_argument("--min-pass", type=int, default=None, + help="Gate passes when at least this many scenarios pass " + "(default: all must pass)") + args = p.parse_args() + + with open(args.scenarios) as f: + scenarios = json.load(f) + skills = { + "humanize": SYSTEM.format(skill=open(args.humanize_skill).read()), + "ai-check": SYSTEM.format(skill=open(args.ai_check_skill).read()), + } + + client = OpenAI(base_url=args.base_url, + api_key=os.environ.get("LLM_API_KEY", "test"), + timeout=600.0, max_retries=0) + + REVISE = ( + "Re-read the skill in your system prompt. The draft below was produced by applying " + "it to a user request. Now perform the skill's OWN verification steps on this draft " + "(for a rewrite: the pre-output gate, self-check, and audit pass; for a report: the " + "skill's required output format), fix every violation those steps find, and output " + "ONLY the corrected final result — no commentary, no list of changes.\n\n" + "Original request:\n{user}\n\nDraft:\n{draft}" + ) + + results, resolved_models = [], set() + for scn in scenarios: + user = f"{scn['user_prompt']}\n\n{scn['input']}" + system = skills[scn["skill"]] + # ai-check scenarios need judgment (register calibration, mixed authorship) that + # the lite tier measurably lacks; route them to the strong model when configured. + models = [args.model] + if scn["skill"] == "ai-check" and args.strong_model: + models = [args.strong_model, args.model] + # Two-pass execution mirrors the skill's draft -> verify -> fix protocol, which a + # single completion cannot perform (it can't revise tokens it already emitted). + # Pass 2 only invokes the skill's own verification steps, so a PR that weakens + # the skill's gates still fails here — the harness adds process, not rules. + out, model = call(client, models, system, user) + resolved_models.add(model) + failures = evaluate(scn, out) + passes = 1 + if failures: + revised, model = call(client, models, system, + REVISE.format(user=user, draft=out)) + resolved_models.add(model) + f2 = evaluate(scn, revised) + passes = 2 + if len(f2) < len(failures): + out, failures = revised, f2 + results.append({"scenario": scn, "output": out, "failures": failures, + "passes": passes}) + status = "PASS" if not failures else f"FAIL ({len(failures)})" + print(f" scenario {scn['id']:>2} [{scn['skill']:8s}] {status} " + f"(passes: {passes}) {scn['name']}", file=sys.stderr) + + failed = [r for r in results if r["failures"]] + n_pass = len(results) - len(failed) + required = args.min_pass if args.min_pass is not None else len(results) + passed = n_pass >= required + resolved = f"; resolved model: `{'`, `'.join(sorted(resolved_models))}`" + + lines = [ + MARKER, + f"## Scenario tests — {'PASS ✅' if passed else 'FAIL ❌'} " + f"({n_pass}/{len(results)} scenarios, gate requires ≥ {required})", + "", + f"_The gate threshold is a **regression floor** calibrated to this executor's " + f"measured ceiling on the current skills — red means a PR scores below today's " + f"behavior, not that {required}/{len(results)} is the quality goal (the goal is " + f"{len(results)}/{len(results)}; frontier agentic executors reach it)._", + "", + f"{len(results)} behavioral fixtures from `tests/SCENARIOS.md`, executed with the " + f"PR's skill files (executor: `{args.model}`{resolved}) and checked against each " + "scenario's objective pass criteria.", + "", + "| # | Scenario | Skill | Result |", + "|---|---|---|---|", + ] + for r in results: + s = r["scenario"] + res = "✅" if not r["failures"] else f"❌ {len(r['failures'])} check(s)" + lines.append(f"| {s['id']} | {s['name']} | `{s['skill']}` | {res} |") + + for cap in (1500, 700, 350): + detail = ["", f"
Per-scenario output ({len(results)} items)" + "", ""] + detail += [block(r["scenario"], r["output"], r["failures"], cap) for r in results] + detail += ["", "
"] + report = "\n".join(lines + detail) + if len(report) < 60000: + break + else: + report = "\n".join(lines) + + with open(args.report, "w") as f: + f.write(report) + with open(args.json_out, "w") as f: + json.dump({"pass": passed, "n_pass": n_pass, "required": required, + "failed": [{"id": r["scenario"]["id"], "failures": r["failures"]} + for r in failed], + "results": [{"id": r["scenario"]["id"], "output": r["output"], + "failures": r["failures"], "passes": r.get("passes")} + for r in results]}, f, indent=1) + print(report) + + +if __name__ == "__main__": + main() diff --git a/.github/benchmark/scenarios.json b/.github/benchmark/scenarios.json new file mode 100644 index 0000000..98da8be --- /dev/null +++ b/.github/benchmark/scenarios.json @@ -0,0 +1,453 @@ +[ + { + "id": 1, + "name": "Flagrant AI prose", + "skill": "humanize", + "user_prompt": "Can you humanize this paragraph?", + "input": "In today's fast-paced world, it is important to note that artificial intelligence has become increasingly pivotal in shaping how organizations operate. Furthermore, AI systems are often utilized to streamline workflows and foster innovation across teams. Moreover, the comprehensive integration of these tools — often regarded as a robust solution — can significantly enhance productivity. It is clear that companies leveraging AI capabilities tend to outperform their peers; however, the implementation requires careful planning. The standard approach: identify use cases, evaluate vendors, and pilot incrementally.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "scenario tells", + "patterns": [ + "in today's", + "it is clear that", + "\\bfoster" + ] + }, + { + "type": "require_regex", + "label": "zero-anchor meta-note appended", + "patterns": [ + "\\[Note:" + ] + } + ] + }, + { + "id": 2, + "name": "ai-check: flagrant AI prose", + "skill": "ai-check", + "user_prompt": "Can you run ai-check on this?", + "input": "In today's fast-paced world, it is important to note that artificial intelligence has become increasingly pivotal in shaping how organizations operate. Furthermore, AI systems are often utilized to streamline workflows and foster innovation across teams. Moreover, the comprehensive integration of these tools — often regarded as a robust solution — can significantly enhance productivity. It is clear that companies leveraging AI capabilities tend to outperform their peers; however, the implementation requires careful planning. The standard approach: identify use cases, evaluate vendors, and pilot incrementally.", + "checks": [ + { + "type": "verdict_in", + "values": [ + "AI" + ] + }, + { + "type": "score_min", + "value": 18 + }, + { + "type": "fraction_in", + "values": [ + "Pure AI", + "Heavily AI-edited" + ] + } + ] + }, + { + "id": 3, + "name": "ai-check: real Slack message (false-positive calibration)", + "skill": "ai-check", + "user_prompt": "score this text", + "input": "ok so the migration is mostly done. ~80% of rows backfilled, the rest are stuck behind a weird FK constraint i didn't know existed. fwiw the constraint was added in 2022 by someone who left, no comments. gonna dig into it tmrw morning.\n\noh also — the staging cluster keeps OOMing during the backfill. bumped the memory limit twice already. lmk if anyone has a better idea than just throwing ram at it", + "checks": [ + { + "type": "verdict_in", + "values": [ + "Human", + "Likely Human" + ] + }, + { + "type": "score_max", + "value": 4 + }, + { + "type": "fraction_in", + "values": [ + "Pure human" + ] + } + ] + }, + { + "id": 4, + "name": "Subtle rhetorical scaffolding", + "skill": "humanize", + "user_prompt": "humanize this", + "input": "The decision to ship the feature flag system wasn't really about velocity. It was about something deeper: the recognition that our deployment process had become a bottleneck more than an asset. Three engineers had quit citing release anxiety. The pattern was clear: we were optimizing for the wrong thing.\n\nWhat we built isn't just a flag service. It's a commitment to deployable-by-default. Faster iteration cycles. Safer rollbacks. Clearer ownership. That's the part that stuck.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "Signal I patterns", + "patterns": [ + "not just", + "isn't just", + "\\bmore \\w+ than\\b", + "the part that stuck" + ] + } + ] + }, + { + "id": 5, + "name": "Long-form essay consistency", + "skill": "humanize", + "user_prompt": "humanize this essay", + "input": "In today's rapidly evolving workplace landscape, remote work has become a pivotal aspect of how organizations operate. Furthermore, the comprehensive integration of digital collaboration tools has fundamentally transformed how teams communicate and coordinate. Moreover, leaders are increasingly leveraging these capabilities to foster innovation across distributed workforces.\n\nIt is important to note that the shift to remote work presents both opportunities and challenges. On one hand, employees benefit from enhanced flexibility and improved work-life balance. On the other hand, organizations must navigate complex issues related to team cohesion, performance management, and cultural alignment.\n\nStudies have shown that successful remote work implementation requires a multifaceted approach. Companies that thrive in this environment typically invest in robust digital infrastructure, foster a culture of trust and accountability, and implement clear communication protocols. By embracing these best practices, organizations can unlock the full potential of distributed work.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "RLHF balance + corporate cliches", + "patterns": [ + "on one hand", + "on the other hand", + "unlock the full potential", + "best practices", + "multifaceted" + ] + }, + { + "type": "length_ratio", + "min": 0.3, + "max": 1.35 + } + ] + }, + { + "id": 6, + "name": "Technical / engineering register", + "skill": "humanize", + "user_prompt": "make this read like an actual engineer wrote it", + "input": "Implementing efficient database indexing is a critical aspect of modern application development. There are several key strategies that developers should consider when designing their indexing approach. First, it is important to analyze query patterns to identify the most frequently accessed columns. Second, composite indexes can significantly improve performance for queries that filter on multiple columns. Third, developers should be mindful of the trade-offs between read performance and write overhead.\n\nA common pitfall is over-indexing, which can lead to degraded write performance and increased storage costs. To avoid this, it is recommended to regularly review index usage statistics and remove indexes that are not contributing to query performance. Additionally, leveraging tools like query analyzers and explain plans can provide valuable insights into how the database engine is utilizing your indexes.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "academic-paper register + enumeration scaffolding", + "patterns": [ + "critical aspect", + "several key strategies", + "valuable insights", + "\\bFirst,", + "\\bSecond,", + "\\bThird," + ] + } + ] + }, + { + "id": 7, + "name": "Professional email", + "skill": "humanize", + "user_prompt": "humanize this email", + "input": "Hi John,\n\nI hope this email finds you well. I wanted to reach out regarding the quarterly report we discussed last week. As we approach the end of the quarter, I think it would be valuable to align on our priorities and ensure we're on the same page.\n\nA few key items I'd love to discuss:\n- The status of the marketing initiatives\n- Budget allocation for the upcoming quarter\n- Any blockers the team might be facing\n\nWould it be possible to schedule a 30-minute call this week? I'm flexible with timing and happy to work around your schedule.\n\nLooking forward to connecting soon.\n\nBest regards,\n[Sender]", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "templated openers/closers", + "patterns": [ + "hope this email finds you well", + "looking forward to connecting", + "align on", + "on the same page" + ] + }, + { + "type": "length_ratio", + "min": 0.25, + "max": 0.9 + } + ] + }, + { + "id": 8, + "name": "Slack register collapse", + "skill": "humanize", + "user_prompt": "rewrite this as a real Slack message", + "input": "hey team! 🚀 quick update on the API migration project. just wanted to share where we are.\n\nwe've made significant progress on the core endpoints, having successfully migrated 75% of the user-facing routes. our team has been working hard to ensure backward compatibility throughout this process. additionally, we've implemented comprehensive testing to validate the new infrastructure.\n\na few things to flag:\n- we'll need to coordinate with the frontend team for the final cutover\n- some edge cases around session management are still being worked through\n- performance benchmarks are looking promising\n\nhappy to discuss further in our standup tomorrow! let me know if you have any questions in the meantime.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "polished status-report residue", + "patterns": [ + "significant progress", + "working hard to ensure", + "\\badditionally\\b", + "happy to discuss", + "let me know if you have any questions" + ] + } + ] + }, + { + "id": 10, + "name": "RLHF helpful-assistant register", + "skill": "humanize", + "user_prompt": "humanize this answer", + "input": "Here's how I'd think about your question. There are a few things to consider. On one hand, there are valid reasons to migrate to PostgreSQL. The community is large, the ecosystem is mature, and you get strong consistency guarantees. On the other hand, MySQL has its own merits — it's been battle-tested at scale, has excellent tooling, and many of your team members likely have more experience with it.\n\nUltimately, the right choice depends on your specific situation. I'd recommend taking some time to evaluate your team's strengths, your performance requirements, and your long-term roadmap. Whatever you decide, the most important thing is to make sure you have buy-in from all stakeholders.\n\nHope this helps! Let me know if you'd like to discuss any of these points further.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "Lever 9 RLHF voice", + "patterns": [ + "here's how i'd think", + "a few things to consider", + "on one hand", + "on the other hand", + "ultimately, the right choice", + "i'd recommend taking", + "whatever you decide", + "hope this helps", + "let me know if" + ] + } + ] + }, + { + "id": 11, + "name": "ai-check: academic abstract (false-positive calibration)", + "skill": "ai-check", + "user_prompt": "is this AI?", + "input": "We present a novel framework for distributed consensus in Byzantine fault-tolerant systems. Our approach extends the PBFT protocol by introducing a probabilistic gossip mechanism that significantly reduces message complexity from O(n²) to O(n log n) under realistic network conditions; we demonstrate this through both formal analysis and empirical evaluation on a 256-node testbed. The proposed protocol maintains the safety guarantees of traditional PBFT while exhibiting markedly improved throughput characteristics. Notably, our experiments show a 4.7x improvement in transaction processing rate compared to baseline implementations, with latency improvements of 23% at the 99th percentile. These results suggest that probabilistic gossip-based consensus represents a promising direction for large-scale Byzantine fault-tolerant deployments. Future work will explore the extension of this framework to dynamic membership scenarios and asynchronous network models.", + "checks": [ + { + "type": "verdict_in", + "values": [ + "Likely Human", + "Uncertain" + ] + }, + { + "type": "fraction_in", + "values": [ + "Pure human", + "Lightly AI-assisted" + ] + } + ] + }, + { + "id": 12, + "name": "ai-check: mixed authorship", + "skill": "ai-check", + "user_prompt": "score this", + "input": "just spent the weekend debugging the new auth flow. turns out the session cookies were being rotated twice on every refresh because someone (me) forgot to mark the middleware as idempotent. classic 2am bug — looks fine in dev, dies under any kind of load.\n\nFurthermore, this experience highlighted the importance of implementing comprehensive middleware testing strategies in our development workflow. It is clear that establishing robust testing patterns from the outset can significantly reduce the likelihood of similar issues arising in production environments.\n\nanyway. the fix is in PR #4471. tests added, will land tomorrow if nothing breaks.", + "checks": [ + { + "type": "verdict_in", + "values": [ + "Uncertain", + "Likely Human" + ] + }, + { + "type": "fraction_in", + "values": [ + "Mixed authorship", + "Lightly AI-assisted" + ] + } + ] + }, + { + "id": 14, + "name": "Creative/lyrical register (literary rationalization)", + "skill": "humanize", + "user_prompt": "/humanize this hopeful passage", + "input": "The city is still dark, but at the edges the sky has started to push back. You've been up since five — not because anything woke you, but because something new won't let you sleep. You make coffee and stand at the window, both hands around the mug, and you think: this is where it starts. Not every door has been tried. Not every version of yourself has been met. The unfinished thing on your desk isn't evidence of failure — it's evidence that you showed up, and the work is still there, patient as dirt, waiting to grow something. You drink your coffee. The sky goes pink at the edges, and the day opens like a door.", + "checks": [ + { + "type": "forbid_regex", + "label": "hard-rule tells (em dash, semicolon, banned vocab, preamble/changelog)", + "patterns": [ + "—", + ";", + "\\bdelve", + "\\bleverag", + "\\butiliz", + "\\brobust\\b", + "\\bcomprehensive\\b", + "\\bstreamlin", + "\\bfurthermore\\b", + "\\bmoreover\\b", + "\\bpivotal\\b", + "it is important to note", + "here is the humanized", + "here's the humanized", + "main moves", + "what i changed" + ] + }, + { + "type": "forbid_regex", + "label": "poetic negation pivot, anaphora, colon reveal", + "patterns": [ + "isn't proof", + "it's proof you", + "not every \\w+ has been", + "you think:" + ] + } + ] + } +] \ No newline at end of file diff --git a/.github/workflows/scenario-tests.yml b/.github/workflows/scenario-tests.yml new file mode 100644 index 0000000..f2937d1 --- /dev/null +++ b/.github/workflows/scenario-tests.yml @@ -0,0 +1,125 @@ +name: scenario-tests + +on: + # Same-repo PRs: workflow and code run from the PR branch (trusted). + pull_request: + paths: + - "humanize/SKILL.md" + - "ai-check/SKILL.md" + - "tests/SCENARIOS.md" + - ".github/benchmark/scenarios.json" + - ".github/benchmark/run_scenarios.py" + - ".github/workflows/scenario-tests.yml" + # Fork PRs: workflow and ALL code run from the base branch; only the fork's skill + # files are fetched, as prompt data. Fork code never executes here. + pull_request_target: + types: [opened, synchronize, reopened, labeled] + paths: + - "humanize/SKILL.md" + - "ai-check/SKILL.md" + workflow_dispatch: + +permissions: + contents: read + pull-requests: write + +concurrency: + group: scenarios-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + BENCH: .github/benchmark + EXECUTOR_MODEL: gemini-flash-lite-latest + # Non-lite tier for the judgment-heavy ai-check scenarios: its free quota is only + # ~20 requests/day, which covers <=8 calls/run; falls back to lite when exhausted. + EXECUTOR_STRONG_MODEL: gemini-flash-latest + EXECUTOR_BASE_URL: https://generativelanguage.googleapis.com/v1beta/openai/ + +jobs: + scenarios: + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) || + (github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name != github.repository) + runs-on: ubuntu-latest + environment: benchmark + timeout-minutes: 45 + steps: + # For pull_request this checks out the PR merge ref; for pull_request_target it + # checks out the BASE branch (safe default) — fork code is never checked out. + - uses: actions/checkout@v4 + + - name: Take the fork PR's skill files as data (fork PRs only) + if: github.event_name == 'pull_request_target' + run: | + git fetch origin "pull/${{ github.event.pull_request.number }}/head" + git checkout FETCH_HEAD -- humanize/SKILL.md ai-check/SKILL.md + echo "Testing fork skills: humanize $(wc -w < humanize/SKILL.md)w, ai-check $(wc -w < ai-check/SKILL.md)w" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install client deps + run: pip install --quiet openai + + - name: Run scenario fixtures + env: + LLM_API_KEY: ${{ secrets.GEMINI_API_KEY }} + run: | + python $BENCH/run_scenarios.py \ + --scenarios $BENCH/scenarios.json \ + --humanize-skill humanize/SKILL.md \ + --ai-check-skill ai-check/SKILL.md \ + --model "$EXECUTOR_MODEL" --strong-model "$EXECUTOR_STRONG_MODEL" --base-url "$EXECUTOR_BASE_URL" \ + --min-pass 6 \ + --report /tmp/scenario-report.md --json-out /tmp/scenario-results.json + + - name: Upload results + if: always() + uses: actions/upload-artifact@v4 + with: + name: scenario-results + path: | + /tmp/scenario-results.json + /tmp/scenario-report.md + overwrite: true + + - name: Post or update PR comment + if: always() && (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + if (!fs.existsSync('/tmp/scenario-report.md')) { + core.warning('No report produced; skipping comment.'); + return; + } + const body = fs.readFileSync('/tmp/scenario-report.md', 'utf8'); + const marker = ''; + const comments = await github.paginate(github.rest.issues.listComments, { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + per_page: 100, + }); + const existing = comments.find(c => c.body && c.body.includes(marker)); + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + - name: Enforce scenario results + run: | + python -c "import json,sys; r=json.load(open('/tmp/scenario-results.json')); sys.exit(0 if r['pass'] else 1)" diff --git a/tests/SCENARIOS.md b/tests/SCENARIOS.md index 0d75783..ef25341 100644 --- a/tests/SCENARIOS.md +++ b/tests/SCENARIOS.md @@ -4,6 +4,12 @@ Regression fixtures for the `humanize` and `ai-check` skills. To verify either s Methodology: writing-skills TDD (RED baseline, then GREEN with skill). +CI coverage: scenarios 1-8, 10-12, and 14 run on every skill-touching PR via the +`scenario-tests` workflow — their objective pass criteria are encoded in +`.github/benchmark/scenarios.json` (keep the two in sync when editing a scenario). +Scenarios 9 (voice matching) and 13 (round-trip) have judgment-based criteria and +remain agent-run only. + Size budget: `humanize/SKILL.md` stays under ~6,000 words (`wc -w`). Research citations and rationale belong in `humanize/references/research.md`, not inline; new patterns should extend the Signal I checklist (single source of truth) rather than duplicating it in a lever.