From d4d7f4efc6e14d33ddd1e1efb53bea06b340a43c Mon Sep 17 00:00:00 2001 From: kiannidev <156195510+kiannidev@users.noreply.github.com> Date: Mon, 6 Jul 2026 05:57:57 +0200 Subject: [PATCH] test(engine): add predicted-gate golden parity runner (#2286) Commit golden snapshots for every predicted-gate fixture and assert the public re-export surface stays byte-identical in CI via test:engine-parity. Co-authored-by: Cursor --- package.json | 3 +- scripts/record-engine-parity-goldens.ts | 25 ++++++++++ test/contract/engine-parity.test.ts | 48 +++++++++++++++++++ .../golden/clean-pass-gittensor.json | 13 +++++ .../golden/clean-pass-oss-anti-slop.json | 16 +++++++ .../golden/duplicate-pr-block.json | 20 ++++++++ .../golden/manifest-blocked-path.json | 13 +++++ .../golden/missing-linked-issue-block.json | 20 ++++++++ .../golden/path-gated-check-with-paths.json | 20 ++++++++ .../path-gated-check-without-paths.json | 13 +++++ .../golden/readiness-warning.json | 20 ++++++++ 11 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 scripts/record-engine-parity-goldens.ts create mode 100644 test/contract/engine-parity.test.ts create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/clean-pass-gittensor.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/clean-pass-oss-anti-slop.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/duplicate-pr-block.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/manifest-blocked-path.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/missing-linked-issue-block.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-with-paths.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-without-paths.json create mode 100644 test/fixtures/engine-parity/predicted-gate/golden/readiness-warning.json diff --git a/package.json b/package.json index 8f01efb5c9..a68ad41a95 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "test:unit": "vitest run test/unit", "test:integration": "vitest run test/integration", "test:upstream-contract": "vitest run test/contract/upstream-contract.test.ts", + "test:engine-parity": "vitest run test/contract/engine-parity.test.ts", "test:changed": "vitest run --changed=origin/main", "test:workers": "vitest run --config vitest.workers.config.ts", "test:coverage": "vitest run --coverage", @@ -73,7 +74,7 @@ "test:smoke:observability": "node scripts/smoke-observability-traces.mjs", "test:smoke:browser:install": "playwright install chromium", "test:smoke:browser": "node scripts/smoke-ui-browser.mjs", - "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", + "test:ci": "git diff --check && npm run actionlint && npm run db:migrations:check && npm run db:schema-drift:check && npm run selfhost:env-reference:check && npm run selfhost:validate-observability && npm run cf-typegen:check && npm run typecheck && npm run test:coverage && npm run test:engine-parity && npm run test:workers && npm run build:mcp && npm run test:mcp-pack && npm run build:miner && npm run test:miner-pack && npm run rees:test && npm run ui:openapi:check && npm run ui:openapi:settings-parity && npm run ui:version-audit && npm run docs:drift-check && npm run command-reference:check && npm run ui:lint && npm run ui:typecheck && npm run ui:test && npm run ui:build", "test:release": "npm run test:ci && npm run changelog:check", "test:release:mcp": "npm run test:ci && npm run changelog:check:mcp", "test:watch": "vitest", diff --git a/scripts/record-engine-parity-goldens.ts b/scripts/record-engine-parity-goldens.ts new file mode 100644 index 0000000000..fa18044d08 --- /dev/null +++ b/scripts/record-engine-parity-goldens.ts @@ -0,0 +1,25 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { buildPredictedGateVerdict } from "../src/rules/predicted-gate.ts"; +import { predictedGateFixtures } from "../test/fixtures/engine-parity/predicted-gate/index.ts"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const goldenDir = join(repoRoot, "test/fixtures/engine-parity/predicted-gate/golden"); + +mkdirSync(goldenDir, { recursive: true }); + +for (const fixture of predictedGateFixtures) { + const verdict = buildPredictedGateVerdict({ + input: fixture.input, + manifest: fixture.manifest, + repo: fixture.repo, + issues: fixture.issues, + pullRequests: fixture.pullRequests, + ...(fixture.changedPaths ? { changedPaths: fixture.changedPaths } : {}), + }); + writeFileSync(join(goldenDir, `${fixture.id}.json`), `${JSON.stringify(verdict, null, 2)}\n`, "utf8"); +} + +process.stdout.write(`Recorded ${predictedGateFixtures.length} predicted-gate goldens in ${goldenDir}\n`); diff --git a/test/contract/engine-parity.test.ts b/test/contract/engine-parity.test.ts new file mode 100644 index 0000000000..35822dbc53 --- /dev/null +++ b/test/contract/engine-parity.test.ts @@ -0,0 +1,48 @@ +/** + * Golden-snapshot parity suite for predicted-gate (#2286). + * + * Each fixture under `test/fixtures/engine-parity/predicted-gate/` is run through the public + * `buildPredictedGateVerdict` re-export surface (`src/rules/predicted-gate.ts`) and compared + * byte-for-byte against a committed golden JSON in `golden/`. This catches silent drift when + * engine extraction or refactors change gate output without updating fixtures. + * + * To intentionally refresh goldens after a deliberate gate-behavior change, run: + * `npx tsx scripts/record-engine-parity-goldens.ts` + */ +import { readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; + +import { buildPredictedGateVerdict } from "../../src/rules/predicted-gate"; +import { predictedGateFixtures } from "../fixtures/engine-parity/predicted-gate"; + +const FIXTURE_DIR = join(process.cwd(), "test", "fixtures", "engine-parity", "predicted-gate"); +const GOLDEN_DIR = join(FIXTURE_DIR, "golden"); + +describe("predicted-gate engine parity (#2286)", () => { + it("has one committed golden per scenario fixture", () => { + const scenarioFiles = readdirSync(FIXTURE_DIR) + .filter((name) => name.endsWith(".ts") && name !== "index.ts" && !name.startsWith("_")) + .sort(); + const goldenFiles = readdirSync(GOLDEN_DIR) + .filter((name) => name.endsWith(".json")) + .sort(); + + expect(goldenFiles).toEqual(scenarioFiles.map((name) => name.replace(/\.ts$/, ".json"))); + expect(predictedGateFixtures).toHaveLength(scenarioFiles.length); + }); + + it.each(predictedGateFixtures)("$id matches the committed golden output", (fixture) => { + const golden = JSON.parse(readFileSync(join(GOLDEN_DIR, `${fixture.id}.json`), "utf8")); + const verdict = buildPredictedGateVerdict({ + input: fixture.input, + manifest: fixture.manifest, + repo: fixture.repo, + issues: fixture.issues, + pullRequests: fixture.pullRequests, + ...(fixture.changedPaths ? { changedPaths: fixture.changedPaths } : {}), + }); + + expect(verdict).toEqual(golden); + }); +}); diff --git a/test/fixtures/engine-parity/predicted-gate/golden/clean-pass-gittensor.json b/test/fixtures/engine-parity/predicted-gate/golden/clean-pass-gittensor.json new file mode 100644 index 0000000000..54fbcce8b6 --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/clean-pass-gittensor.json @@ -0,0 +1,13 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "success", + "title": "Gittensory Orb Review Agent passed", + "summary": "No configured hard blocker was found. Advisory findings, if any, stay advisory.", + "readinessScore": 95, + "blockers": [], + "warnings": [], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. Provide the PR's changed paths to also predict the focus-manifest path policy, the size/guardrail hold, and any pre-merge check scoped to changed paths; without them only path-independent title/description/label pre-merge checks are predicted. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/clean-pass-oss-anti-slop.json b/test/fixtures/engine-parity/predicted-gate/golden/clean-pass-oss-anti-slop.json new file mode 100644 index 0000000000..9784386e94 --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/clean-pass-oss-anti-slop.json @@ -0,0 +1,16 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "oss-anti-slop", + "conclusion": "success", + "title": "Gittensory Orb Review Agent passed", + "summary": "No configured hard blocker was found. Advisory findings, if any, stay advisory.", + "readinessScore": 95, + "blockers": [], + "warnings": [], + "funnel": { + "message": "This repo runs the Gittensor anti-slop gate. Gittensor lets GitHub contributors earn for open-source work like this — register to start earning.", + "registerUrl": "https://gittensor.io" + }, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. Provide the PR's changed paths to also predict the focus-manifest path policy, the size/guardrail hold, and any pre-merge check scoped to changed paths; without them only path-independent title/description/label pre-merge checks are predicted. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/duplicate-pr-block.json b/test/fixtures/engine-parity/predicted-gate/golden/duplicate-pr-block.json new file mode 100644 index 0000000000..3c265fd5e3 --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/duplicate-pr-block.json @@ -0,0 +1,20 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "failure", + "title": "Gittensory Orb Review Agent: Linked issue overlaps another open PR", + "summary": "Linked issue overlaps another open PR — Review the related PRs before spending reviewer time on duplicate work.", + "readinessScore": 69, + "blockers": [ + { + "code": "duplicate_pr_risk", + "title": "Linked issue overlaps another open PR", + "detail": "Other open pull requests reference the same linked issue set: #42.", + "action": "Review the related PRs before spending reviewer time on duplicate work." + } + ], + "warnings": [], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. Provide the PR's changed paths to also predict the focus-manifest path policy, the size/guardrail hold, and any pre-merge check scoped to changed paths; without them only path-independent title/description/label pre-merge checks are predicted. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/manifest-blocked-path.json b/test/fixtures/engine-parity/predicted-gate/golden/manifest-blocked-path.json new file mode 100644 index 0000000000..64342a1f5f --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/manifest-blocked-path.json @@ -0,0 +1,13 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "success", + "title": "Gittensory Orb Review Agent passed", + "summary": "No configured hard blocker was found. Advisory findings, if any, stay advisory.", + "readinessScore": 95, + "blockers": [], + "warnings": [], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. The size-hold prediction uses changed FILE count only, not changed LINE count (line-diff stats are not available pre-submission), so it may under-predict a hold for a PR with many changed lines across few files. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/missing-linked-issue-block.json b/test/fixtures/engine-parity/predicted-gate/golden/missing-linked-issue-block.json new file mode 100644 index 0000000000..e855350471 --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/missing-linked-issue-block.json @@ -0,0 +1,20 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "failure", + "title": "Gittensory Orb Review Agent: No linked issue detected", + "summary": "No linked issue detected — If this PR is intended to solve an issue, link it explicitly in the PR body.", + "readinessScore": 80, + "blockers": [ + { + "code": "missing_linked_issue", + "title": "No linked issue detected", + "detail": "No closing reference or linked issue number was found in the PR metadata/body.", + "action": "If this PR is intended to solve an issue, link it explicitly in the PR body." + } + ], + "warnings": [], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. Provide the PR's changed paths to also predict the focus-manifest path policy, the size/guardrail hold, and any pre-merge check scoped to changed paths; without them only path-independent title/description/label pre-merge checks are predicted. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-with-paths.json b/test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-with-paths.json new file mode 100644 index 0000000000..55ac2417b6 --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-with-paths.json @@ -0,0 +1,20 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "failure", + "title": "Gittensory Orb Review Agent: Pre-merge check not satisfied: Tests for src", + "summary": "Pre-merge check not satisfied: Tests for src — Update the PR to satisfy the check, then re-run the gate.", + "readinessScore": 95, + "blockers": [ + { + "code": "pre_merge_check_required", + "title": "Pre-merge check not satisfied: Tests for src", + "detail": "This PR does not satisfy the maintainer pre-merge check \"Tests for src\": the title must contain \"ZZZ-never\".", + "action": "Update the PR to satisfy the check, then re-run the gate." + } + ], + "warnings": [], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. The size-hold prediction uses changed FILE count only, not changed LINE count (line-diff stats are not available pre-submission), so it may under-predict a hold for a PR with many changed lines across few files. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-without-paths.json b/test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-without-paths.json new file mode 100644 index 0000000000..54fbcce8b6 --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/path-gated-check-without-paths.json @@ -0,0 +1,13 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "success", + "title": "Gittensory Orb Review Agent passed", + "summary": "No configured hard blocker was found. Advisory findings, if any, stay advisory.", + "readinessScore": 95, + "blockers": [], + "warnings": [], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. Provide the PR's changed paths to also predict the focus-manifest path policy, the size/guardrail hold, and any pre-merge check scoped to changed paths; without them only path-independent title/description/label pre-merge checks are predicted. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +} diff --git a/test/fixtures/engine-parity/predicted-gate/golden/readiness-warning.json b/test/fixtures/engine-parity/predicted-gate/golden/readiness-warning.json new file mode 100644 index 0000000000..794742826c --- /dev/null +++ b/test/fixtures/engine-parity/predicted-gate/golden/readiness-warning.json @@ -0,0 +1,20 @@ +{ + "predicted": true, + "basis": "public_config", + "pack": "gittensor", + "conclusion": "success", + "title": "Gittensory Orb Review Agent passed", + "summary": "No configured hard blocker was found. Advisory findings, if any, stay advisory.", + "readinessScore": 80, + "blockers": [], + "warnings": [ + { + "code": "readiness_score_below_threshold", + "title": "Readiness score is below the configured threshold", + "detail": "The public readiness score is 80/100, below the repository threshold of 90/100.", + "action": "Use the readiness panel as advisory maintainer context; the score does not block this PR." + } + ], + "funnel": null, + "note": "Predicted from the repo's public .gittensory.yml gate config + safe defaults. The maintainer may have private dashboard overrides not reflected here, and the dual-model AI-consensus blocker is only evaluated on a real PR. The slop score is NOT evaluated pre-submission (it needs the diff content) and may still fail the real gate. Provide the PR's changed paths to also predict the focus-manifest path policy, the size/guardrail hold, and any pre-merge check scoped to changed paths; without them only path-independent title/description/label pre-merge checks are predicted. Every author is gated the same: a configured hard blocker fails the gate regardless of confirmed-contributor status (which affects only on-chain scoring)." +}