diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99112bf8..4010829e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,10 +72,11 @@ jobs: - name: cargo test run: cargo test - # P-022 step 7a (#260) — COMPARE MODE over the committed corpus. Not #260's - # acceptance: the five-repository sweep and the large-solution controls are - # separate work, and nothing here may be read as shadow mode having been - # achieved. + # P-022 step 7a (#260) — COMPARE MODE over the committed corpus: the FAST half + # of #260's test matrix, and one leg of it. The five pinned OSS repositories, + # the large-solution controls and the examples tree are the scheduled/manual + # half and live in `.github/workflows/shadow-sweep.yml`. Nothing here may be + # read as shadow mode having been achieved. # # What it gates: every committed facts document through BOTH engines on # byte-attested same input, failing on any acceptance-`unexplained` diff --git a/.github/workflows/shadow-sweep.yml b/.github/workflows/shadow-sweep.yml new file mode 100644 index 00000000..3b35ef5b --- /dev/null +++ b/.github/workflows/shadow-sweep.yml @@ -0,0 +1,371 @@ +# The scheduled/manual half of #260's CI strategy: compare mode over the five +# pinned OSS repositories of #243, the large/multi-project solution controls, +# the examples tree, and the Windows path forms. The fast PR gate (the +# committed corpus and the C# samples) lives in ci.yml and is unchanged. +# +# THE RULES THIS WORKFLOW EXISTS TO ENFORCE, none of which a green tick alone +# would show: +# +# * a target is checked out AT ITS PIN and `git rev-parse HEAD` is compared +# with it before anything runs — a target that moved is a failed job, not a +# newer measurement; +# * the extractor runs EXACTLY ONCE per document (`own-check.sh +# --emit-facts` persists the file stage 1 already wrote), and the driver +# reads that file once, as bytes; +# * `OWN_SHADOW_ENGINE` is always set explicitly and the result names the +# adapter by sha256, so a stale build cannot stand in; +# * zero documents compared is a FAILURE, and so is a declared target the run +# never reached — the aggregation re-checks both against the committed +# sweep definition rather than trusting the legs' own exit codes. +# +# Reproduction artifacts upload on failure only (#260's CI strategy). The run +# summary and the per-document results upload always: they are the record. +name: shadow sweep (#260) + +on: + workflow_dispatch: + schedule: + # Weekly, Mondays 04:17 UTC. Off the hour on purpose — the top of the hour + # is the busiest slot on shared runners. + - cron: "17 4 * * 1" + +permissions: + contents: read + +env: + DOCUMENT_OUT: shadow-sweep + +jobs: + # One job per DOCUMENT rather than per repository: the directory walk and the + # `.sln` fan-out are different extractor paths over the same checkout, they + # fail differently, and a matrix leg that covered both could not say which + # half a failure belonged to. + document: + name: ${{ matrix.id }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - id: ShareX.repo + target: ShareX + repository: ShareX/ShareX + commit: 0df9ca4d83eed9d2489048c539d7d1fc2860fdec + mode: directory-walk + input: "" + - id: ShareX.sln + target: ShareX + repository: ShareX/ShareX + commit: 0df9ca4d83eed9d2489048c539d7d1fc2860fdec + mode: solution + input: ShareX.sln + - id: MahApps.Metro.repo + target: MahApps.Metro + repository: MahApps/MahApps.Metro + commit: 72099e310bac2d12ac98fd7560b69679252519f5 + mode: directory-walk + input: "" + - id: MahApps.Metro.sln + target: MahApps.Metro + repository: MahApps/MahApps.Metro + commit: 72099e310bac2d12ac98fd7560b69679252519f5 + mode: solution + input: src/MahApps.Metro.sln + - id: MaterialDesignInXamlToolkit.repo + target: MaterialDesignInXamlToolkit + repository: MaterialDesignInXAML/MaterialDesignInXamlToolkit + commit: ef3a5ea434e39182b1848f5e11aaea6b3890581f + mode: directory-walk + input: "" + - id: AvalonEdit.repo + target: AvalonEdit + repository: icsharpcode/AvalonEdit + commit: ed0bd149059469ac9bd39b13cf8a341b12a6c1da + mode: directory-walk + input: "" + - id: AvalonEdit.sln + target: AvalonEdit + repository: icsharpcode/AvalonEdit + commit: ed0bd149059469ac9bd39b13cf8a341b12a6c1da + mode: solution + input: ICSharpCode.AvalonEdit.sln + - id: ClosedXML.repo + target: ClosedXML + repository: ClosedXML/ClosedXML + commit: 4e89dcedd83cad553e84d2d97f77fc3d7deb630f + mode: directory-walk + input: "" + - id: ClosedXML.sln + target: ClosedXML + repository: ClosedXML/ClosedXML + commit: 4e89dcedd83cad553e84d2d97f77fc3d7deb630f + mode: solution + input: ClosedXML.sln + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + # The target, AT ITS PIN. `persist-credentials: false` because nothing in + # this job pushes anywhere and a third-party checkout has no business + # carrying a token. + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + repository: ${{ matrix.repository }} + ref: ${{ matrix.commit }} + path: targets/${{ matrix.target }} + persist-credentials: false + - name: Verify the pin + # Drift is a hard failure of this leg, never a warning: a target that + # moved has not been measured, it has been replaced. + run: | + head=$(git -C "targets/${{ matrix.target }}" rev-parse HEAD) + echo "${{ matrix.target }} HEAD=$head pin=${{ matrix.commit }}" + test "$head" = "${{ matrix.commit }}" || { + echo "PIN DRIFT: ${{ matrix.target }} is at $head, the sweep pins ${{ matrix.commit }}" + exit 1 + } + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + with: + dotnet-version: "8.0.x" + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master, 2026-07-10 + with: + toolchain: stable + # The WindowsDesktop reference pack, materialized exactly the way the + # corpus-benchmark job does it. NOT continue-on-error here: the pinned + # targets are WPF/WinForms, so a run without the refs measures a + # different document than the sweep says it measured. + - name: Materialize framework reference assemblies + run: | + tmp=$(mktemp -d) + printf '%s\n' \ + '' \ + ' ' \ + ' net8.0-windows' \ + ' true' \ + ' true' \ + ' true' \ + ' ' \ + '' > "$tmp/ref.csproj" + dotnet restore "$tmp/ref.csproj" >/dev/null + d=$(find "$HOME/.nuget/packages/microsoft.windowsdesktop.app.ref" -type d -name 'net8.0' | sort | tail -1) + test -n "$d" || { echo "the WindowsDesktop reference pack did not resolve"; exit 1; } + echo "OWN_EXTRA_REF_DIRS=$d" >> "$GITHUB_ENV" + echo "framework refs: $d ($(find "$d" -name '*.dll' | wc -l) dlls)" + - name: Build the dev-only engine adapter + working-directory: rust + run: cargo build --release -p own-shadow --bin own-shadow-engine + # ONE extraction. `--emit-facts` persists the file stage 1 already wrote; + # stage 2 is the reference's verdict path and is neither a second + # extraction nor the comparison. + - name: Extract the OwnIR facts, exactly once + run: | + input="targets/${{ matrix.target }}" + if [ -n "${{ matrix.input }}" ]; then + input="targets/${{ matrix.target }}/${{ matrix.input }}" + fi + echo "extracting: $input" + scripts/own-check.sh --format sarif --severity warning \ + --emit-facts "$RUNNER_TEMP/facts.json" -- "$input" \ + > "$RUNNER_TEMP/findings.sarif.json" + ls -l "$RUNNER_TEMP/facts.json" + - name: Describe the document + run: | + python - <<'PY' + import hashlib, json, os + facts = os.path.join(os.environ["RUNNER_TEMP"], "facts.json") + with open(facts, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + target = "${{ matrix.target }}" + rel = "${{ matrix.input }}" + where = f"targets/{target}" + (f"/{rel}" if rel else "") + manifest = { + "schema": 1, + "targets": [target], + "documents": [{ + "id": "${{ matrix.id }}", + "source": facts, + "target": target, + "target_commit": "${{ matrix.commit }}", + "extraction_mode": "${{ matrix.mode }}", + "extraction_command": ( + "OWN_EXTRA_REF_DIRS= " + "scripts/own-check.sh --format sarif --severity warning " + f"--emit-facts -- {where}"), + "facts_sha256": digest, + "timeout_seconds": 600.0, + }], + } + with open(os.path.join(os.environ["RUNNER_TEMP"], "manifest.json"), "w") as f: + json.dump(manifest, f, indent=2) + print(json.dumps(manifest, indent=2)) + PY + - name: Compare mode over the document + env: + OWN_SHADOW_ENGINE: ${{ github.workspace }}/rust/target/release/own-shadow-engine + run: | + python scripts/shadow_compare.py --engine compare \ + --manifest "$RUNNER_TEMP/manifest.json" \ + --out "$RUNNER_TEMP/$DOCUMENT_OUT" + # The record uploads whether or not the run agreed; a leg that only + # uploaded on failure would leave the aggregation unable to tell a green + # target from a skipped one. + - name: Upload the run record + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: shadow-sweep-${{ matrix.id }} + path: ${{ runner.temp }}/${{ env.DOCUMENT_OUT }} + retention-days: 30 + if-no-files-found: warn + + # The examples tree: cheap, named in #260's test matrix, and the one document + # of this sweep that needs no third-party checkout. + examples: + name: examples + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4 + with: + dotnet-version: "8.0.x" + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master, 2026-07-10 + with: + toolchain: stable + - name: Build the dev-only engine adapter + working-directory: rust + run: cargo build --release -p own-shadow --bin own-shadow-engine + - name: Extract the OwnIR facts, exactly once + run: | + scripts/own-check.sh --format sarif --severity warning \ + --emit-facts "$RUNNER_TEMP/facts.json" -- examples \ + > "$RUNNER_TEMP/findings.sarif.json" + - name: Describe the document + run: | + python - <<'PY' + import hashlib, json, os + facts = os.path.join(os.environ["RUNNER_TEMP"], "facts.json") + with open(facts, "rb") as f: + digest = hashlib.sha256(f.read()).hexdigest() + manifest = { + "schema": 1, + "targets": ["examples"], + "documents": [{ + "id": "examples", + "source": facts, + "target": "examples", + "target_commit": os.environ["GITHUB_SHA"], + "extraction_mode": "directory-walk", + "extraction_command": ( + "scripts/own-check.sh --format sarif --severity warning " + "--emit-facts -- examples"), + "facts_sha256": digest, + "timeout_seconds": 600.0, + }], + } + with open(os.path.join(os.environ["RUNNER_TEMP"], "manifest.json"), "w") as f: + json.dump(manifest, f, indent=2) + PY + - name: Compare mode over the examples document + env: + OWN_SHADOW_ENGINE: ${{ github.workspace }}/rust/target/release/own-shadow-engine + run: | + python scripts/shadow_compare.py --engine compare \ + --manifest "$RUNNER_TEMP/manifest.json" \ + --out "$RUNNER_TEMP/$DOCUMENT_OUT" + - name: Upload the run record + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: shadow-sweep-examples + path: ${{ runner.temp }}/${{ env.DOCUMENT_OUT }} + retention-days: 30 + if-no-files-found: warn + + # Windows path forms, MEASURED and nothing promised beyond what runs: the + # committed-corpus gate, on a Windows runner, with an adapter built there. + # The driver's own controls run too and cannot skip — that group is the one + # that had never executed on Windows at all. + windows-path-forms: + name: windows path forms (committed corpus) + runs-on: windows-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + - uses: dtolnay/rust-toolchain@fa04a1451ff1842e2626ccb99004d0195b455a88 # master, 2026-07-10 + with: + toolchain: stable + - name: Build the dev-only engine adapter + working-directory: rust + run: cargo build --release -p own-shadow --bin own-shadow-engine + - name: Compare mode over the committed corpus + env: + OWN_SHADOW_ENGINE: ${{ github.workspace }}/rust/target/release/own-shadow-engine.exe + run: python scripts/shadow_compare.py --engine compare --corpus --quiet --out "$RUNNER_TEMP/shadow" + - name: The compare driver's controls (adapter required) + env: + OWN_SHADOW_ENGINE: ${{ github.workspace }}/rust/target/release/own-shadow-engine.exe + OWN_SHADOW_COMPARE_REQUIRED: "1" + run: python tests/test_shadow_compare.py + - name: Upload the divergence reports + if: failure() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: shadow-sweep-windows-reports + path: ${{ runner.temp }}/shadow + retention-days: 14 + if-no-files-found: ignore + + # The aggregation, and the reason it exists: every leg above can be green and + # the SWEEP still be a lie, because a leg that never ran uploads nothing and a + # matrix that lost an entry says nothing at all. This job assembles one run + # record from every leg and checks it against the committed sweep DEFINITION — + # the denominator lives there, not in whatever happened to be measured. + aggregate: + name: aggregate the sweep + runs-on: ubuntu-latest + needs: [document, examples, windows-path-forms] + if: always() + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: "3.13" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: shadow-sweep-* + path: ${{ runner.temp }}/legs + - name: Every leg must have run + run: | + echo "document: ${{ needs.document.result }}" + echo "examples: ${{ needs.examples.result }}" + echo "windows path forms: ${{ needs['windows-path-forms'].result }}" + fail=0 + for r in "${{ needs.document.result }}" "${{ needs.examples.result }}" \ + "${{ needs['windows-path-forms'].result }}"; do + [ "$r" = "success" ] || fail=1 + done + test "$fail" -eq 0 || { echo "a leg of the sweep did not succeed"; exit 1; } + - name: Assemble one run record + run: | + python tests/shadow_sweep.py --collect "$RUNNER_TEMP/legs" \ + --write "$RUNNER_TEMP/p022-shadow-sweep.result.json" \ + --workflow-run-url "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" + - name: Check it against the committed sweep definition + run: python tests/shadow_sweep.py --result "$RUNNER_TEMP/p022-shadow-sweep.result.json" + - name: Upload the run summary + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: shadow-sweep-run-summary + path: ${{ runner.temp }}/p022-shadow-sweep.result.json + retention-days: 90 + if-no-files-found: warn diff --git a/docs/evidence/p022-shadow-acc-1.result.json b/docs/evidence/p022-shadow-acc-1.result.json index 3f2055d4..95db54b3 100644 --- a/docs/evidence/p022-shadow-acc-1.result.json +++ b/docs/evidence/p022-shadow-acc-1.result.json @@ -4,9 +4,9 @@ "campaign": "p022-shadow-acc-1", "definition": "docs/evidence/p022-shadow-acc-1.json", "definition_sha256": "031ec599db9edd3e7be1cbfd8e99f582550c291c43acbe8197a6846d95153e39", - "source_commit": "4c1c9d81972171c3d9c8c1d814debb73a4e471f5", + "source_commit": "1c6a611f1ed8f3a7883896c497c2f443fd0f174c", "dirty": false, - "recorded_at": "2026-09-07T14:47:36Z", + "recorded_at": "2026-09-08T01:58:34Z", "layers": [ "python", "python-variants", @@ -18,7 +18,7 @@ "id": "M00", "outcome": "survived", "catchers": [], - "elapsed_seconds": 7.7 + "elapsed_seconds": 19.7 }, "mutations": [ { @@ -27,10 +27,11 @@ "catchers": [ "python-compare::compare-control", "python-compare::compare-one-read", + "python-compare::compare-summary", "python::artifact-golden", "python::capture-verify" ], - "elapsed_seconds": 6.1 + "elapsed_seconds": 8.9 }, { "id": "A02", @@ -38,7 +39,7 @@ "catchers": [ "rust/tests/engine.rs::this_engine_reproduces_its_committed_capture" ], - "elapsed_seconds": 7.3 + "elapsed_seconds": 10.8 }, { "id": "A03", @@ -46,7 +47,7 @@ "catchers": [ "python::carry-foreign" ], - "elapsed_seconds": 7.3 + "elapsed_seconds": 10.9 }, { "id": "A04", @@ -54,7 +55,7 @@ "catchers": [ "python::carry-foreign" ], - "elapsed_seconds": 7.0 + "elapsed_seconds": 10.9 }, { "id": "A05", @@ -62,7 +63,7 @@ "catchers": [ "python::structural-control" ], - "elapsed_seconds": 7.3 + "elapsed_seconds": 10.7 }, { "id": "A06", @@ -70,7 +71,7 @@ "catchers": [ "rust/src/lib.rs::base64::tests::a_non_canonical_encoding_is_refused" ], - "elapsed_seconds": 7.5 + "elapsed_seconds": 10.8 }, { "id": "A07", @@ -83,7 +84,7 @@ "rust/tests/engine.rs::this_engine_reproduces_its_committed_capture", "rust/tests/repro.rs::every_committed_artifact_round_trips_and_verifies" ], - "elapsed_seconds": 7.7 + "elapsed_seconds": 10.6 }, { "id": "A08", @@ -91,7 +92,7 @@ "catchers": [ "python::structural-control" ], - "elapsed_seconds": 7.6 + "elapsed_seconds": 10.9 }, { "id": "A09", @@ -99,7 +100,7 @@ "catchers": [ "rust/tests/repro.rs::verify_refuses_each_structural_violation" ], - "elapsed_seconds": 7.6 + "elapsed_seconds": 10.8 }, { "id": "A10", @@ -108,7 +109,7 @@ "python-compare::compare-control", "python::structural-control" ], - "elapsed_seconds": 7.3 + "elapsed_seconds": 10.8 }, { "id": "A11", @@ -116,7 +117,7 @@ "catchers": [ "rust/tests/repro.rs::verify_refuses_each_structural_violation" ], - "elapsed_seconds": 7.9 + "elapsed_seconds": 10.7 }, { "id": "A12", @@ -124,7 +125,7 @@ "catchers": [ "python-variants::variant-ledger" ], - "elapsed_seconds": 7.6 + "elapsed_seconds": 10.8 } ] } diff --git a/docs/evidence/p022-shadow-acc-2.json b/docs/evidence/p022-shadow-acc-2.json index e306fb25..fbfce085 100644 --- a/docs/evidence/p022-shadow-acc-2.json +++ b/docs/evidence/p022-shadow-acc-2.json @@ -211,8 +211,8 @@ "description": "the driver stops failing the run when an engine crashes", "rule": "R-2: a crash is a run-level hard failure, never a fallback", "target": "scripts/shadow_compare.py", - "pattern": " worst = max\\(worst, EXIT\\[OUTCOME_EXECUTION_FAILURE\\]\\)", - "replacement": " worst = max(worst, EXIT[OUTCOME_AGREED])", + "pattern": " return OUTCOME_EXECUTION_FAILURE, report, elapsed", + "replacement": " return OUTCOME_AGREED, report, elapsed", "expected_catchers": [ "python-compare::compare-control" ] @@ -222,8 +222,8 @@ "description": "the driver hands the port bytes it did not hand the reference (one trailing space — the same document, a different byte sequence)", "rule": "R-1/B-1: one read, and BOTH engines get that same buffer — the observable form of a second read", "target": "scripts/shadow_compare.py", - "pattern": " port = run_port\\(raw, binary, timeout\\)", - "replacement": " port = run_port(raw + b\" \", binary, timeout)", + "pattern": " port = run_port\\(raw, adapter, timeout\\)", + "replacement": " port = run_port(raw + b\" \", adapter, timeout)", "expected_catchers": [ "python-compare::compare-one-read" ] diff --git a/docs/evidence/p022-shadow-acc-2.result.json b/docs/evidence/p022-shadow-acc-2.result.json index 682c27c2..52f52897 100644 --- a/docs/evidence/p022-shadow-acc-2.result.json +++ b/docs/evidence/p022-shadow-acc-2.result.json @@ -3,10 +3,10 @@ "comment": "Recorded mutation-campaign run (scripts/mutate_campaign.py --run). Raw facts only: outcomes, catchers, provenance. Counts are derived by scripts/render_checkpoint_status.py; regenerate this file by re-running the campaign, never by hand.", "campaign": "p022-shadow-acc-2", "definition": "docs/evidence/p022-shadow-acc-2.json", - "definition_sha256": "a7cb74b7bb120c34a6d139d067f55fee738e3eb00b15b023604de964d392e223", - "source_commit": "4c1c9d81972171c3d9c8c1d814debb73a4e471f5", + "definition_sha256": "49ba5fbd28b7e0ab3b0829aa1e82ae2ac11840926e6c30546f6d458d3c3b8cee", + "source_commit": "1c6a611f1ed8f3a7883896c497c2f443fd0f174c", "dirty": false, - "recorded_at": "2026-09-07T14:49:53Z", + "recorded_at": "2026-09-08T02:01:52Z", "layers": [ "python", "python-compare", @@ -17,7 +17,7 @@ "id": "M00", "outcome": "survived", "catchers": [], - "elapsed_seconds": 7.3 + "elapsed_seconds": 10.7 }, "mutations": [ { @@ -28,7 +28,7 @@ "python::reduction-scope", "python::verdict-pairing" ], - "elapsed_seconds": 5.8 + "elapsed_seconds": 8.8 }, { "id": "B02", @@ -36,7 +36,7 @@ "catchers": [ "python::reduction-golden" ], - "elapsed_seconds": 6.9 + "elapsed_seconds": 10.5 }, { "id": "B03", @@ -44,7 +44,7 @@ "catchers": [ "rust/tests/reduce.rs::every_reduction_golden_is_reproduced_byte_for_byte" ], - "elapsed_seconds": 7.1 + "elapsed_seconds": 10.7 }, { "id": "B04", @@ -52,7 +52,7 @@ "catchers": [ "python::boundary-policy" ], - "elapsed_seconds": 7.2 + "elapsed_seconds": 10.7 }, { "id": "B05", @@ -60,7 +60,7 @@ "catchers": [ "python::boundary-policy" ], - "elapsed_seconds": 7.0 + "elapsed_seconds": 10.8 }, { "id": "B06", @@ -69,7 +69,7 @@ "rust/tests/reduce.rs::every_reduction_golden_is_reproduced_byte_for_byte", "rust/tests/reduce.rs::the_boundary_policy_explains_exactly_what_it_names" ], - "elapsed_seconds": 7.7 + "elapsed_seconds": 10.6 }, { "id": "B07", @@ -78,7 +78,7 @@ "python::boundary-policy", "python::reduction-golden" ], - "elapsed_seconds": 7.4 + "elapsed_seconds": 10.6 }, { "id": "B08", @@ -87,7 +87,7 @@ "rust/tests/reduce.rs::every_reduction_golden_is_reproduced_byte_for_byte", "rust/tests/reduce.rs::the_boundary_policy_explains_exactly_what_it_names" ], - "elapsed_seconds": 7.4 + "elapsed_seconds": 10.8 }, { "id": "B09", @@ -97,7 +97,7 @@ "python::reduction-control", "python::verdict-pairing" ], - "elapsed_seconds": 7.4 + "elapsed_seconds": 10.7 }, { "id": "B10", @@ -108,7 +108,7 @@ "rust/tests/reduce.rs::the_reducer_is_silent_on_unchanged_data_and_names_a_synthetic_divergence", "rust/tests/reduce.rs::the_same_fields_in_a_different_key_order_are_a_difference" ], - "elapsed_seconds": 7.4 + "elapsed_seconds": 10.6 }, { "id": "B11", @@ -117,7 +117,7 @@ "python::trace-golden", "python::trace-shape" ], - "elapsed_seconds": 7.6 + "elapsed_seconds": 10.6 }, { "id": "B12", @@ -125,16 +125,18 @@ "catchers": [ "rust/tests/trace.rs::every_trace_golden_is_reproduced_byte_for_byte" ], - "elapsed_seconds": 7.8 + "elapsed_seconds": 10.7 }, { "id": "B13", "outcome": "caught", "catchers": [ + "python-compare::compare-control", "python-compare::compare-one-read", + "python-compare::compare-summary", "python::artifact-golden" ], - "elapsed_seconds": 7.1 + "elapsed_seconds": 10.5 }, { "id": "B14", @@ -142,7 +144,7 @@ "catchers": [ "rust/tests/engine.rs::this_engine_reproduces_its_committed_capture" ], - "elapsed_seconds": 7.4 + "elapsed_seconds": 10.9 }, { "id": "B15", @@ -151,24 +153,26 @@ "python-compare::", "python::derived-control" ], - "elapsed_seconds": 7.3 + "elapsed_seconds": 7.8 }, { "id": "B16", "outcome": "caught", "catchers": [ + "python-compare::compare-adapter-identity", "python-compare::compare-control" ], - "elapsed_seconds": 7.0 + "elapsed_seconds": 10.7 }, { "id": "B17", "outcome": "caught", "catchers": [ "python-compare::compare-control", - "python-compare::compare-one-read" + "python-compare::compare-one-read", + "python-compare::compare-summary" ], - "elapsed_seconds": 7.3 + "elapsed_seconds": 10.6 }, { "id": "B18", @@ -176,7 +180,7 @@ "catchers": [ "python-compare::compare-control" ], - "elapsed_seconds": 7.5 + "elapsed_seconds": 10.5 } ] } diff --git a/docs/evidence/p022-shadow-sweep-1.json b/docs/evidence/p022-shadow-sweep-1.json new file mode 100644 index 00000000..42113f44 --- /dev/null +++ b/docs/evidence/p022-shadow-sweep-1.json @@ -0,0 +1,141 @@ +{ + "schema": 1, + "campaign": "p022-shadow-sweep-1", + "description": "The mutation campaign for #260's FINAL-ACCEPTANCE sweep: the compare driver's version-2 surfaces (the adapter's identity, the manifest run with its provenance and denominators, the empty-set rule) and tests/shadow_sweep.py, the one interpreter that decides whether a recorded run is evidence. Every mutation edits a production surface — the driver or the interpreter — never a test, and both layers run for every mutation (discipline 3: no fail-fast). The mutations are chosen to be exactly the ways a sweep could report agreement it never measured: an identity taken from the wrong file, a stale digest waved through, an empty run called agreement, unexplained counted as agreed, and a skipped target that fails nothing. M00 is the harness-honesty control.", + "layers": [ + { + "id": "python-compare", + "cwd": ".", + "command": [ + "python3", + "tests/test_shadow_compare.py" + ], + "parser": "python-fail" + }, + { + "id": "python-sweep", + "cwd": ".", + "command": [ + "python3", + "tests/test_shadow_sweep.py" + ], + "parser": "python-fail" + } + ], + "control": { + "id": "M00", + "description": "harness-honesty control: no mutation at all, which must report zero failing layers" + }, + "mutations": [ + { + "id": "S01", + "description": "the adapter's identity is taken from a file other than the one that runs", + "rule": "a recorded comparison names the ENGINE THAT RAN, by digest", + "target": "scripts/shadow_compare.py", + "pattern": " with open\\(binary, \"rb\"\\) as f:\n raw = f\\.read\\(\\)", + "replacement": " with open(__file__, \"rb\") as f:\n raw = f.read()", + "expected_catchers": [ + "python-compare::compare-adapter-identity" + ] + }, + { + "id": "S02", + "description": "a manifest whose facts_sha256 does not match the file is waved through", + "rule": "every document is verified against the manifest BEFORE any engine runs", + "target": "scripts/shadow_compare.py", + "pattern": " if digest != doc\\.facts_sha256:", + "replacement": " if False and digest != doc.facts_sha256:", + "expected_catchers": [ + "python-compare::compare-manifest-digest" + ] + }, + { + "id": "S03", + "description": "an empty manifest is no longer refused where the rule is stated", + "rule": "#250's fifth failure mode: a run that compared zero documents FAILS", + "target": "scripts/shadow_compare.py", + "pattern": " if not documents:\n problems\\.append\\(\"the manifest lists no documents, so this run compared \"\n \"ZERO of them — a failure, never agreement\"\\)", + "replacement": " if not documents:\n pass", + "expected_catchers": [ + "python-compare::compare-empty-set" + ] + }, + { + "id": "S04", + "description": "the run summary counts every document as agreed, whatever it was", + "rule": "the denominators are derived from the OUTCOMES, never from the entries", + "target": "scripts/shadow_compare.py", + "pattern": " row\\[_TALLY_FIELD\\[outcome\\]\\] \\+= 1", + "replacement": " row[\"agreed\"] += 1", + "expected_catchers": [ + "python-compare::compare-summary" + ] + }, + { + "id": "S05", + "description": "the driver stops failing a run for a declared target it never reached", + "rule": "a target with compare-attempted 0 is a FAILED target, never a passed repository", + "target": "scripts/shadow_compare.py", + "pattern": " empty = \\[t for t, row in sorted\\(per_target\\.items\\(\\)\\)\n if row\\[\"compare_attempted\"\\] == 0\\]", + "replacement": " empty: list[str] = []", + "expected_catchers": [ + "python-compare::compare-skipped-target" + ] + }, + { + "id": "S06", + "description": "the aggregation stops failing for a declared target missing from the run", + "rule": "the DEFINITION is the denominator; a target nobody measured did not pass", + "target": "tests/shadow_sweep.py", + "pattern": " if row is None:\n problems\\.append\\(\n f\"target \\{target!r\\} is declared by the definition and missing \"\n f\"from the run — a skipped target is not a passed repository\"\\)\n continue", + "replacement": " if row is None:\n continue", + "expected_catchers": [ + "python-sweep::shadow-sweep-interpreter" + ] + }, + { + "id": "S07", + "description": "the aggregation accepts a document outcome other than agreed", + "rule": "only 'agreed' is an outcome this sweep may report", + "target": "tests/shadow_sweep.py", + "pattern": " if outcome != OUTCOME_AGREED:", + "replacement": " if False and outcome != OUTCOME_AGREED:", + "expected_catchers": [ + "python-sweep::shadow-sweep-interpreter" + ] + }, + { + "id": "S08", + "description": "the aggregation stops seeing an unexplained observation inside an agreed document", + "rule": "an outcome that contradicts its own observations is a finding, not a rounding", + "target": "tests/shadow_sweep.py", + "pattern": " if unexplained:", + "replacement": " if False and unexplained:", + "expected_catchers": [ + "python-sweep::shadow-sweep-interpreter" + ] + }, + { + "id": "S09", + "description": "the aggregation stops noticing a document the definition never declared", + "rule": "the definition may not be a subset of what happened to be measured", + "target": "tests/shadow_sweep.py", + "pattern": " for doc_id in sorted\\(set\\(measured\\) - set\\(declared\\)\\):", + "replacement": " for doc_id in sorted(set() - set(declared)):", + "expected_catchers": [ + "python-sweep::shadow-sweep-interpreter" + ] + }, + { + "id": "S10", + "description": "the aggregation stops checking that the totals are the sum of their parts", + "rule": "a total that is not the sum of its rows is a number somebody typed", + "target": "tests/shadow_sweep.py", + "pattern": " if totals\\[name\\] != recomputed\\[name\\]:", + "replacement": " if False and totals[name] != recomputed[name]:", + "expected_catchers": [ + "python-sweep::shadow-sweep-interpreter" + ] + } + ] +} diff --git a/docs/evidence/p022-shadow-sweep-1.result.json b/docs/evidence/p022-shadow-sweep-1.result.json new file mode 100644 index 00000000..efb53aa0 --- /dev/null +++ b/docs/evidence/p022-shadow-sweep-1.result.json @@ -0,0 +1,104 @@ +{ + "schema": 1, + "comment": "Recorded mutation-campaign run (scripts/mutate_campaign.py --run). Raw facts only: outcomes, catchers, provenance. Counts are derived by scripts/render_checkpoint_status.py; regenerate this file by re-running the campaign, never by hand.", + "campaign": "p022-shadow-sweep-1", + "definition": "docs/evidence/p022-shadow-sweep-1.json", + "definition_sha256": "2edb73d9e343b70f2be00f5e041d6431e61fa058048d73d5d083f6286ae8a42d", + "source_commit": "565de6d49f3d38decbd9117723a39691dcb54aee", + "dirty": false, + "recorded_at": "2026-09-08T02:03:50Z", + "layers": [ + "python-compare", + "python-sweep" + ], + "command": "every layer the definition declares, for every mutation", + "control": { + "id": "M00", + "outcome": "survived", + "catchers": [], + "elapsed_seconds": 8.8 + }, + "mutations": [ + { + "id": "S01", + "outcome": "caught", + "catchers": [ + "python-compare::compare-adapter-identity", + "python-compare::compare-summary" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S02", + "outcome": "caught", + "catchers": [ + "python-compare::compare-manifest-digest" + ], + "elapsed_seconds": 9.0 + }, + { + "id": "S03", + "outcome": "caught", + "catchers": [ + "python-compare::compare-empty-set" + ], + "elapsed_seconds": 8.9 + }, + { + "id": "S04", + "outcome": "caught", + "catchers": [ + "python-compare::compare-summary" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S05", + "outcome": "caught", + "catchers": [ + "python-compare::compare-skipped-target" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S06", + "outcome": "caught", + "catchers": [ + "python-sweep::shadow-sweep-interpreter" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S07", + "outcome": "caught", + "catchers": [ + "python-sweep::shadow-sweep-interpreter" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S08", + "outcome": "caught", + "catchers": [ + "python-sweep::shadow-sweep-interpreter" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S09", + "outcome": "caught", + "catchers": [ + "python-sweep::shadow-sweep-interpreter" + ], + "elapsed_seconds": 8.8 + }, + { + "id": "S10", + "outcome": "caught", + "catchers": [ + "python-sweep::shadow-sweep-interpreter" + ], + "elapsed_seconds": 8.8 + } + ] +} diff --git a/docs/evidence/p022-shadow-sweep.json b/docs/evidence/p022-shadow-sweep.json new file mode 100644 index 00000000..860f97be --- /dev/null +++ b/docs/evidence/p022-shadow-sweep.json @@ -0,0 +1,181 @@ +{ + "schema": 1, + "comment": "The definition of #260's final-acceptance sweep: what SHOULD be measured. It is the DENOMINATOR — a run that reports only what it happened to measure cannot be short. The recorded run beside it is one actual execution; a re-run replaces that file whole. Read by tests/shadow_sweep.py, rendered by scripts/render_checkpoint_status.py.", + "sweep": "p022-shadow-sweep", + "driver_version": 2, + "default_timeout_seconds": 600.0, + "adapter_build_command": "cd rust && cargo build --release -p own-shadow --bin own-shadow-engine", + "driver_command": "OWN_SHADOW_ENGINE= python scripts/shadow_compare.py --engine compare --manifest --out ", + "reference_pack": "microsoft.windowsdesktop.app.ref ref/net8.0, materialized by a scratch net8.0-windows UseWPF/UseWindowsForms/EnableWindowsTargeting project + dotnet restore and exported as OWN_EXTRA_REF_DIRS — the mechanism ci.yml's corpus-benchmark job uses", + "pin_rule": "A checkout materializes the COMMIT (a depth-1 fetch of the sha, detached) and `git rev-parse HEAD` is compared with the pin before anything runs. Drift is a hard failure of that target, never a newer measurement.", + "targets": [ + { + "target": "ShareX", + "upstream": "ShareX/ShareX", + "pinned_commit": "0df9ca4d83eed9d2489048c539d7d1fc2860fdec", + "solutions": [ + { + "path": "ShareX.sln", + "csproj_members": 13, + "cs_files_under_members": 1126, + "taken": true + }, + { + "path": "ShareX.ImageEditor.sln", + "csproj_members": 2, + "cs_files_under_members": 433, + "taken": false + } + ], + "slnx": [] + }, + { + "target": "MahApps.Metro", + "upstream": "MahApps/MahApps.Metro", + "pinned_commit": "72099e310bac2d12ac98fd7560b69679252519f5", + "solutions": [ + { + "path": "src/MahApps.Metro.sln", + "csproj_members": 4, + "cs_files_under_members": 328, + "taken": true + } + ], + "slnx": [] + }, + { + "target": "MaterialDesignInXamlToolkit", + "upstream": "MaterialDesignInXAML/MaterialDesignInXamlToolkit", + "pinned_commit": "ef3a5ea434e39182b1848f5e11aaea6b3890581f", + "solutions": [], + "slnx": [ + "MaterialDesignToolkit.Full.slnx" + ], + "note": "No classic solution at this pin: the repository carries MaterialDesignToolkit.Full.slnx, the XML solution format, and the extractor's solution resolver reads the classic Project(\"{...}\") = \"name\", \"path\" form only. Teaching it .slnx would be a production change, so this target is covered by its directory walk alone and the record says so." + }, + { + "target": "AvalonEdit", + "upstream": "icsharpcode/AvalonEdit", + "pinned_commit": "ed0bd149059469ac9bd39b13cf8a341b12a6c1da", + "solutions": [ + { + "path": "ICSharpCode.AvalonEdit.sln", + "csproj_members": 3, + "cs_files_under_members": 248, + "taken": true + }, + { + "path": "ICSharpCode.AvalonEdit.Documentation.sln", + "csproj_members": 1, + "cs_files_under_members": 214, + "taken": false + } + ], + "slnx": [] + }, + { + "target": "ClosedXML", + "upstream": "ClosedXML/ClosedXML", + "pinned_commit": "4e89dcedd83cad553e84d2d97f77fc3d7deb630f", + "solutions": [ + { + "path": "ClosedXML.sln", + "csproj_members": 6, + "cs_files_under_members": 921, + "taken": true + } + ], + "slnx": [] + }, + { + "target": "examples", + "upstream": "PhysShell/Own.NET", + "pinned_commit": null, + "solutions": [], + "slnx": [], + "note": "The examples tree of this repository. Its pin is the run's own Own.NET commit, which is why the definition carries null here and the interpreter checks the recorded pin against the run's source_commit rather than against a fixed sha." + } + ], + "documents": [ + { + "id": "ShareX.repo", + "target": "ShareX", + "target_commit": "0df9ca4d83eed9d2489048c539d7d1fc2860fdec", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ShareX", + "timeout_seconds": 600.0 + }, + { + "id": "ShareX.sln", + "target": "ShareX", + "target_commit": "0df9ca4d83eed9d2489048c539d7d1fc2860fdec", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ShareX/ShareX.sln", + "timeout_seconds": 600.0 + }, + { + "id": "MahApps.Metro.repo", + "target": "MahApps.Metro", + "target_commit": "72099e310bac2d12ac98fd7560b69679252519f5", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/MahApps.Metro", + "timeout_seconds": 600.0 + }, + { + "id": "MahApps.Metro.sln", + "target": "MahApps.Metro", + "target_commit": "72099e310bac2d12ac98fd7560b69679252519f5", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/MahApps.Metro/src/MahApps.Metro.sln", + "timeout_seconds": 600.0 + }, + { + "id": "MaterialDesignInXamlToolkit.repo", + "target": "MaterialDesignInXamlToolkit", + "target_commit": "ef3a5ea434e39182b1848f5e11aaea6b3890581f", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/MaterialDesignInXamlToolkit", + "timeout_seconds": 600.0 + }, + { + "id": "AvalonEdit.repo", + "target": "AvalonEdit", + "target_commit": "ed0bd149059469ac9bd39b13cf8a341b12a6c1da", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/AvalonEdit", + "timeout_seconds": 600.0 + }, + { + "id": "AvalonEdit.sln", + "target": "AvalonEdit", + "target_commit": "ed0bd149059469ac9bd39b13cf8a341b12a6c1da", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/AvalonEdit/ICSharpCode.AvalonEdit.sln", + "timeout_seconds": 600.0 + }, + { + "id": "ClosedXML.repo", + "target": "ClosedXML", + "target_commit": "4e89dcedd83cad553e84d2d97f77fc3d7deb630f", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ClosedXML", + "timeout_seconds": 600.0 + }, + { + "id": "ClosedXML.sln", + "target": "ClosedXML", + "target_commit": "4e89dcedd83cad553e84d2d97f77fc3d7deb630f", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ClosedXML/ClosedXML.sln", + "timeout_seconds": 600.0 + }, + { + "id": "examples", + "target": "examples", + "target_commit": null, + "extraction_mode": "directory-walk", + "extraction_command": "scripts/own-check.sh --format sarif --severity warning --emit-facts -- examples", + "timeout_seconds": 600.0 + } + ] +} diff --git a/docs/evidence/p022-shadow-sweep.result.json b/docs/evidence/p022-shadow-sweep.result.json new file mode 100644 index 00000000..948e0b4c --- /dev/null +++ b/docs/evidence/p022-shadow-sweep.result.json @@ -0,0 +1,455 @@ +{ + "schema": 1, + "comment": "One recorded run of the #260 sweep, assembled by tests/shadow_sweep.py --collect from what scripts/shadow_compare.py wrote. Raw facts only: identities, outcomes, denominators, provenance. Counts are derived by scripts/render_checkpoint_status.py; regenerate this file by re-running the sweep, never by hand.", + "sweep": "p022-shadow-sweep", + "definition": "docs/evidence/p022-shadow-sweep.json", + "definition_sha256": "9edfaedd3ba1d1f918cd0f386f33b856f473fa5f2c1cbd4025265ecc06eacd74", + "source_commit": "565de6d49f3d38decbd9117723a39691dcb54aee", + "recorded_at": "2026-09-08T02:04:21Z", + "host": "Windows AMD64", + "workflow_run_url": null, + "driver_version": 2, + "adapters": [ + { + "sha256": "94eda8fa31dd8c09443d60904d25c85b2670b584fc732bd787a8ea0458f5d9ef", + "bytes": 1614336 + } + ], + "documents": [ + { + "id": "AvalonEdit.repo", + "source": "AvalonEdit.repo.facts.json", + "target": "AvalonEdit", + "target_commit": "ed0bd149059469ac9bd39b13cf8a341b12a6c1da", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/AvalonEdit", + "facts_sha256": "f6e704f252f2dfcd421ab04e34abc301b1ca736965e7cc98ac81b92e1397a504", + "raw": { + "algorithm": "sha256", + "digest": "f6e704f252f2dfcd421ab04e34abc301b1ca736965e7cc98ac81b92e1397a504", + "bytes": 34507 + }, + "canonical": { + "algorithm": "sha256", + "digest": "623af0f59c614bb28a78c02e83c7ccbdb3f6af3c9c493e17e5598f2ad76ec48a", + "bytes": 22130 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.016 + }, + { + "id": "AvalonEdit.sln", + "source": "AvalonEdit.sln.facts.json", + "target": "AvalonEdit", + "target_commit": "ed0bd149059469ac9bd39b13cf8a341b12a6c1da", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/AvalonEdit/ICSharpCode.AvalonEdit.sln", + "facts_sha256": "a5f2e5b70ff154acbe9a800d291295e34992662d3998ce4f990483bb3cafda2c", + "raw": { + "algorithm": "sha256", + "digest": "a5f2e5b70ff154acbe9a800d291295e34992662d3998ce4f990483bb3cafda2c", + "bytes": 34507 + }, + "canonical": { + "algorithm": "sha256", + "digest": "c3515fceb38647a4e1b335221b54e3a04e76e860ac4188d06e14229c598c24fa", + "bytes": 22130 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.016 + }, + { + "id": "ClosedXML.repo", + "source": "ClosedXML.repo.facts.json", + "target": "ClosedXML", + "target_commit": "4e89dcedd83cad553e84d2d97f77fc3d7deb630f", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ClosedXML", + "facts_sha256": "bbefbfb2089ea0b1ebb2f8a7144c8cbc294ba859fdae6335a72aea58ab8bfc63", + "raw": { + "algorithm": "sha256", + "digest": "bbefbfb2089ea0b1ebb2f8a7144c8cbc294ba859fdae6335a72aea58ab8bfc63", + "bytes": 201623 + }, + "canonical": { + "algorithm": "sha256", + "digest": "4599cfe379cde82307f10a869d778f11abc56eaedbe0d35aeff680dce81778ed", + "bytes": 109227 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.109 + }, + { + "id": "ClosedXML.sln", + "source": "ClosedXML.sln.facts.json", + "target": "ClosedXML", + "target_commit": "4e89dcedd83cad553e84d2d97f77fc3d7deb630f", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ClosedXML/ClosedXML.sln", + "facts_sha256": "ec045c96b3940790ca7a80a3a772f9a831abe98b9834a87b1273f83660dbeb52", + "raw": { + "algorithm": "sha256", + "digest": "ec045c96b3940790ca7a80a3a772f9a831abe98b9834a87b1273f83660dbeb52", + "bytes": 201623 + }, + "canonical": { + "algorithm": "sha256", + "digest": "c6aabbe11592ff7d41723098df9b663bae08052f8cbb70d33a2bfd91ffb61317", + "bytes": 109227 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.109 + }, + { + "id": "MahApps.Metro.repo", + "source": "MahApps.Metro.repo.facts.json", + "target": "MahApps.Metro", + "target_commit": "72099e310bac2d12ac98fd7560b69679252519f5", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/MahApps.Metro", + "facts_sha256": "f6a22fecf1bbb63eac26d72cdcce6db620206c32da90f78ef96fa038c81333b8", + "raw": { + "algorithm": "sha256", + "digest": "f6a22fecf1bbb63eac26d72cdcce6db620206c32da90f78ef96fa038c81333b8", + "bytes": 44712 + }, + "canonical": { + "algorithm": "sha256", + "digest": "a1f0d30bcad70d8267fc9d9abfa1688a8406c2955f59ed789e2ac3e9fb539021", + "bytes": 28765 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.031 + }, + { + "id": "MahApps.Metro.sln", + "source": "MahApps.Metro.sln.facts.json", + "target": "MahApps.Metro", + "target_commit": "72099e310bac2d12ac98fd7560b69679252519f5", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/MahApps.Metro/src/MahApps.Metro.sln", + "facts_sha256": "97f644b3b58a3eea26aa39d1a1d8aa22917e4e7317d4bd20a08ee657eb16d0e3", + "raw": { + "algorithm": "sha256", + "digest": "97f644b3b58a3eea26aa39d1a1d8aa22917e4e7317d4bd20a08ee657eb16d0e3", + "bytes": 44712 + }, + "canonical": { + "algorithm": "sha256", + "digest": "d9ffd61f879be57026e145239d6d90dfa5ac506635afd575990d2940ef9fd8ca", + "bytes": 28765 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.016 + }, + { + "id": "MaterialDesignInXamlToolkit.repo", + "source": "MaterialDesignInXamlToolkit.repo.facts.json", + "target": "MaterialDesignInXamlToolkit", + "target_commit": "ef3a5ea434e39182b1848f5e11aaea6b3890581f", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/MaterialDesignInXamlToolkit", + "facts_sha256": "c0fa752cbe2dc0ce73d49ae8df56ebf676f844c3bb409e897405b36811c83ad3", + "raw": { + "algorithm": "sha256", + "digest": "c0fa752cbe2dc0ce73d49ae8df56ebf676f844c3bb409e897405b36811c83ad3", + "bytes": 53007 + }, + "canonical": { + "algorithm": "sha256", + "digest": "3fd3416e502adff71f07de87489351cb2d3b7faf599ee879b1dd2b7c475c0c18", + "bytes": 35466 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.031 + }, + { + "id": "ShareX.repo", + "source": "ShareX.repo.facts.json", + "target": "ShareX", + "target_commit": "0df9ca4d83eed9d2489048c539d7d1fc2860fdec", + "extraction_mode": "directory-walk", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ShareX", + "facts_sha256": "8d94ea78b8ff26a45e41f5341d678de9912a1ea401a7d702451f4f1fa2a91f8a", + "raw": { + "algorithm": "sha256", + "digest": "8d94ea78b8ff26a45e41f5341d678de9912a1ea401a7d702451f4f1fa2a91f8a", + "bytes": 249448 + }, + "canonical": { + "algorithm": "sha256", + "digest": "f3a30f10ac17dd830e756f751110e732d17898e6f0f6307fa9a436d6a5d1a851", + "bytes": 128850 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.187 + }, + { + "id": "ShareX.sln", + "source": "ShareX.sln.facts.json", + "target": "ShareX", + "target_commit": "0df9ca4d83eed9d2489048c539d7d1fc2860fdec", + "extraction_mode": "solution", + "extraction_command": "OWN_EXTRA_REF_DIRS= scripts/own-check.sh --format sarif --severity warning --emit-facts -- targets/ShareX/ShareX.sln", + "facts_sha256": "f01fa1d3d9ef4ae9a08e45890ff5f8d49dc6d05730edfa58018a581a912601de", + "raw": { + "algorithm": "sha256", + "digest": "f01fa1d3d9ef4ae9a08e45890ff5f8d49dc6d05730edfa58018a581a912601de", + "bytes": 249448 + }, + "canonical": { + "algorithm": "sha256", + "digest": "3a2c3c5eea1cebd612b22bbc1a30cb46a8db39787f6b80c30a5d9d048561c8db", + "bytes": 128850 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.094 + }, + { + "id": "examples", + "source": "examples.facts.json", + "target": "examples", + "target_commit": "565de6d49f3d38decbd9117723a39691dcb54aee", + "extraction_mode": "directory-walk", + "extraction_command": "scripts/own-check.sh --format sarif --severity warning --emit-facts -- examples", + "facts_sha256": "01e617d820d76af1f78057494db4cf065e2d845a84f683b9ce1d574f288f0050", + "raw": { + "algorithm": "sha256", + "digest": "01e617d820d76af1f78057494db4cf065e2d845a84f683b9ce1d574f288f0050", + "bytes": 13291 + }, + "canonical": { + "algorithm": "sha256", + "digest": "322523ad9073af842ebcb0a0f8480abfb267f22404de47dc55f37eb01808bf71", + "bytes": 6705 + }, + "outcome": "agreed", + "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "by_kind": { + "left-only": 0, + "right-only": 0, + "changed": 0, + "ordering-only": 0, + "status": 0, + "projection": 0, + "missing-layer": 0 + }, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", + "wall_clock_seconds": 0.016 + } + ], + "targets": [ + { + "target": "AvalonEdit", + "documents_extracted": 2, + "compare_attempted": 2, + "agreed": 2, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + }, + { + "target": "ClosedXML", + "documents_extracted": 2, + "compare_attempted": 2, + "agreed": 2, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + }, + { + "target": "MahApps.Metro", + "documents_extracted": 2, + "compare_attempted": 2, + "agreed": 2, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + }, + { + "target": "MaterialDesignInXamlToolkit", + "documents_extracted": 1, + "compare_attempted": 1, + "agreed": 1, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + }, + { + "target": "ShareX", + "documents_extracted": 2, + "compare_attempted": 2, + "agreed": 2, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + }, + { + "target": "examples", + "documents_extracted": 1, + "compare_attempted": 1, + "agreed": 1, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + } + ], + "totals": { + "documents_extracted": 10, + "compare_attempted": 10, + "agreed": 10, + "diverged": 0, + "execution_failures": 0, + "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0 + } +} diff --git a/docs/generated/p022-shadow-census.md b/docs/generated/p022-shadow-census.md index 4927ea41..d6d34f20 100644 --- a/docs/generated/p022-shadow-census.md +++ b/docs/generated/p022-shadow-census.md @@ -2,12 +2,13 @@ # P-022 step 7a — shadow-mode infrastructure: census -**Compare mode over the committed corpus — not #260's acceptance.** What is -measured here is every document this repository commits, at all three layers -and on the derived SARIF surface, on byte-attested same input. #260's test -matrix additionally names a five-repository sweep and large-solution controls; -neither is run here, and nothing below may be read as shadow mode having been -achieved, as P-022 being done, or as Rust being the default. +**Compare mode over the committed corpus — one leg of #260's test matrix.** +What is measured here is every document this repository commits, at all three +layers and on the derived SARIF surface, on byte-attested same input. The five +pinned OSS repositories, the large-solution controls and the examples tree are +the OTHER legs and have their own record +([`p022-shadow-sweep.md`](p022-shadow-sweep.md)); nothing below may be read as shadow +mode having been achieved, as P-022 being done, or as Rust being the default. This document is the **live view** of the slice as it stands; the recorded mutation campaigns are their own fragment diff --git a/docs/generated/p022-shadow-mutations.md b/docs/generated/p022-shadow-mutations.md index e126dcad..f8c245cb 100644 --- a/docs/generated/p022-shadow-mutations.md +++ b/docs/generated/p022-shadow-mutations.md @@ -1,4 +1,4 @@ - + # P-022 step 7a — shadow-mode infrastructure: mutation campaigns @@ -163,7 +163,7 @@ Definition: `docs/evidence/p022-shadow-acc-1.json` (sha256 `031ec599db9edd3e…` | measure | value | |--------------------------------------------------|---| -| recorded at commit | `4c1c9d81972171c3d9c8c1d814debb73a4e471f5` | +| recorded at commit | `1c6a611f1ed8f3a7883896c497c2f443fd0f174c` | | layers run (every one, for every mutation) | `python`, `python-variants`, `python-compare`, `rust` | | mutations | 12 | | caught | 12 | @@ -176,7 +176,7 @@ Definition: `docs/evidence/p022-shadow-acc-1.json` (sha256 `031ec599db9edd3e…` | id | rule | mutation | outcome | caught by | |---|---|---|---|---| -| A01 | B-2: the raw identity is taken BEFORE a decode or a parse | the reference hashes the canonical form instead of the bytes it read | caught | `python-compare::compare-control`
`python-compare::compare-one-read`
`python::artifact-golden`
`python::capture-verify` | +| A01 | B-2: the raw identity is taken BEFORE a decode or a parse | the reference hashes the canonical form instead of the bytes it read | caught | `python-compare::compare-control`
`python-compare::compare-one-read`
`python-compare::compare-summary`
`python::artifact-golden`
`python::capture-verify` | | A02 | B-2: the raw identity is taken BEFORE a decode or a parse | the port hashes the canonical form instead of the bytes it read | caught | `rust/tests/engine.rs::this_engine_reproduces_its_committed_capture` | | A03 | B-3: every consumed comes from a run of the engine that claims it | the writer PROMOTES a pre-v3 foreign entry by inventing a consumed | caught | `python::carry-foreign` | | A04 | B-3: an entry that attests nothing is dropped, never carried | the writer carries a pre-v3 foreign entry through unchanged | caught | `python::carry-foreign` | @@ -193,11 +193,11 @@ Definition: `docs/evidence/p022-shadow-acc-1.json` (sha256 `031ec599db9edd3e…` Campaign `p022-shadow-acc-2` — The mutation campaign for #260's acceptance work, part 2: the reduction SCOPE, the boundary POLICY, the DERIVED surface and the compare DRIVER (owner decisions D-4, D-5, D-6, D-7, R-1, R-2). Every mutation edits a production surface — ownlang/repro.py, rust/crates/own-shadow/src/ or scripts/shadow_compare.py — never a test, and every rule that exists on both sides is attacked on both. Three layers run for every mutation (discipline 3: no fail-fast). The policy mutations are the ones this campaign exists for: a class is not a token that excuses whatever it is pinned to, so OD-1 attached to a `changed` or a `projection` observation must stay RED. M00 is the harness-honesty control. -Definition: `docs/evidence/p022-shadow-acc-2.json` (sha256 `a7cb74b7bb120c34…`, 18 mutations). Replay on a clean tree with `python scripts/mutate_campaign.py --campaign docs/evidence/p022-shadow-acc-2.json --run`; the recorded run is raw outcomes and provenance, the counts below are derived from it. +Definition: `docs/evidence/p022-shadow-acc-2.json` (sha256 `49ba5fbd28b7e0ab…`, 18 mutations). Replay on a clean tree with `python scripts/mutate_campaign.py --campaign docs/evidence/p022-shadow-acc-2.json --run`; the recorded run is raw outcomes and provenance, the counts below are derived from it. | measure | value | |--------------------------------------------------|---| -| recorded at commit | `4c1c9d81972171c3d9c8c1d814debb73a4e471f5` | +| recorded at commit | `1c6a611f1ed8f3a7883896c497c2f443fd0f174c` | | layers run (every one, for every mutation) | `python`, `python-compare`, `rust` | | mutations | 18 | | caught | 18 | @@ -222,9 +222,41 @@ Definition: `docs/evidence/p022-shadow-acc-2.json` (sha256 `a7cb74b7bb120c34…` | B10 | D-5: a content observation and a missing layer are ALWAYS unexplained | the port judges every content observation — missing-layer included — a declared boundary | caught | `rust/tests/reduce.rs::a_duplicate_address_permutation_is_changed_on_both_ordinals`
`rust/tests/reduce.rs::the_boundary_policy_explains_exactly_what_it_names`
`rust/tests/reduce.rs::the_reducer_is_silent_on_unchanged_data_and_names_a_synthetic_divergence`
`rust/tests/reduce.rs::the_same_fields_in_a_different_key_order_are_a_difference` | | B11 | D-7: the `~n` suffix is part of the pairing address | the reference drops the duplicate-address ordinal | caught | `python::trace-golden`
`python::trace-shape` | | B12 | D-7: the `~n` suffix is part of the pairing address | the port drops the duplicate-address ordinal | caught | `rust/tests/trace.rs::every_trace_golden_is_reproduced_byte_for_byte` | -| B13 | D-6: the derived identity is of the document THIS engine rendered | the reference's derived digest names the verdict layer, not the SARIF it rendered | caught | `python-compare::compare-one-read`
`python::artifact-golden` | +| B13 | D-6: the derived identity is of the document THIS engine rendered | the reference's derived digest names the verdict layer, not the SARIF it rendered | caught | `python-compare::compare-control`
`python-compare::compare-one-read`
`python-compare::compare-summary`
`python::artifact-golden` | | B14 | D-6: the derived identity is of the document THIS engine rendered | the port's derived digest names the verdict layer, not the SARIF it rendered | caught | `rust/tests/engine.rs::this_engine_reproduces_its_committed_capture` | | B15 | D-6: a renderer-only divergence is two engines' digests differing | the derived comparison compares each engine's digest against itself | caught | `python-compare::`
`python::derived-control` | -| B16 | R-2: a crash is a run-level hard failure, never a fallback | the driver stops failing the run when an engine crashes | caught | `python-compare::compare-control` | -| B17 | R-1/B-1: one read, and BOTH engines get that same buffer — the observable form of a second read | the driver hands the port bytes it did not hand the reference (one trailing space — the same document, a different byte sequence) | caught | `python-compare::compare-control`
`python-compare::compare-one-read` | +| B16 | R-2: a crash is a run-level hard failure, never a fallback | the driver stops failing the run when an engine crashes | caught | `python-compare::compare-adapter-identity`
`python-compare::compare-control` | +| B17 | R-1/B-1: one read, and BOTH engines get that same buffer — the observable form of a second read | the driver hands the port bytes it did not hand the reference (one trailing space — the same document, a different byte sequence) | caught | `python-compare::compare-control`
`python-compare::compare-one-read`
`python-compare::compare-summary` | | B18 | R-2/D-2: two engines disagreeing about whether the input is a document is the owner's decision, not a comparison result | the driver stops treating an input disagreement as a domain decision | caught | `python-compare::compare-control` | + +## final acceptance — the driver's v2 surfaces and the sweep interpreter + +Campaign `p022-shadow-sweep-1` — The mutation campaign for #260's FINAL-ACCEPTANCE sweep: the compare driver's version-2 surfaces (the adapter's identity, the manifest run with its provenance and denominators, the empty-set rule) and tests/shadow_sweep.py, the one interpreter that decides whether a recorded run is evidence. Every mutation edits a production surface — the driver or the interpreter — never a test, and both layers run for every mutation (discipline 3: no fail-fast). The mutations are chosen to be exactly the ways a sweep could report agreement it never measured: an identity taken from the wrong file, a stale digest waved through, an empty run called agreement, unexplained counted as agreed, and a skipped target that fails nothing. M00 is the harness-honesty control. + +Definition: `docs/evidence/p022-shadow-sweep-1.json` (sha256 `2edb73d9e343b70f…`, 10 mutations). Replay on a clean tree with `python scripts/mutate_campaign.py --campaign docs/evidence/p022-shadow-sweep-1.json --run`; the recorded run is raw outcomes and provenance, the counts below are derived from it. + +| measure | value | +|--------------------------------------------------|---| +| recorded at commit | `565de6d49f3d38decbd9117723a39691dcb54aee` | +| layers run (every one, for every mutation) | `python-compare`, `python-sweep` | +| mutations | 10 | +| caught | 10 | +| survived | 0 | +| compile-error (no evidence either way) | 0 | +| invalid-mutation | 0 | +| runner-error | 0 | +| caught without every expected catcher | none | +| honesty control `M00` (unmutated tree must pass) | survived — as required | + +| id | rule | mutation | outcome | caught by | +|---|---|---|---|---| +| S01 | a recorded comparison names the ENGINE THAT RAN, by digest | the adapter's identity is taken from a file other than the one that runs | caught | `python-compare::compare-adapter-identity`
`python-compare::compare-summary` | +| S02 | every document is verified against the manifest BEFORE any engine runs | a manifest whose facts_sha256 does not match the file is waved through | caught | `python-compare::compare-manifest-digest` | +| S03 | #250's fifth failure mode: a run that compared zero documents FAILS | an empty manifest is no longer refused where the rule is stated | caught | `python-compare::compare-empty-set` | +| S04 | the denominators are derived from the OUTCOMES, never from the entries | the run summary counts every document as agreed, whatever it was | caught | `python-compare::compare-summary` | +| S05 | a target with compare-attempted 0 is a FAILED target, never a passed repository | the driver stops failing a run for a declared target it never reached | caught | `python-compare::compare-skipped-target` | +| S06 | the DEFINITION is the denominator; a target nobody measured did not pass | the aggregation stops failing for a declared target missing from the run | caught | `python-sweep::shadow-sweep-interpreter` | +| S07 | only 'agreed' is an outcome this sweep may report | the aggregation accepts a document outcome other than agreed | caught | `python-sweep::shadow-sweep-interpreter` | +| S08 | an outcome that contradicts its own observations is a finding, not a rounding | the aggregation stops seeing an unexplained observation inside an agreed document | caught | `python-sweep::shadow-sweep-interpreter` | +| S09 | the definition may not be a subset of what happened to be measured | the aggregation stops noticing a document the definition never declared | caught | `python-sweep::shadow-sweep-interpreter` | +| S10 | a total that is not the sum of its rows is a number somebody typed | the aggregation stops checking that the totals are the sum of their parts | caught | `python-sweep::shadow-sweep-interpreter` | diff --git a/docs/generated/p022-shadow-sweep.md b/docs/generated/p022-shadow-sweep.md new file mode 100644 index 00000000..4188e4c1 --- /dev/null +++ b/docs/generated/p022-shadow-sweep.md @@ -0,0 +1,62 @@ + + +# P-022 step 7a (#260) — the final-acceptance sweep + +The measurement the acceptance surfaces over the committed corpus deliberately +did not take: the five pinned OSS repositories of #243 at their pinned commits, +the large/multi-project solution controls, and the `examples/` tree. Every +document is one OwnIR byte sequence fed to BOTH engines and judged by compare +mode; the record is [`docs/evidence/p022-shadow-sweep.json`](../evidence/p022-shadow-sweep.json) +(what should be measured) and its `.result.json` (one actual run), read by +`tests/shadow_sweep.py`. Every document agreed and no observation is acceptance-unexplained. + +**What "covered" means here.** A repository is not covered because extraction +succeeded, and a solution is not covered because some project inside it emitted +OwnIR. Coverage is a recorded, non-empty set of documents fed byte-identically +to both engines and judged — per target, with the denominator on the record. +A run that compared zero documents is a failure, and so is a declared target +nothing reached. + +## The run + +| what | value | +|---|---| +| Own.NET commit | `565de6d49f3d38decbd9117723a39691dcb54aee` | +| recorded at | 2026-09-08T02:04:21Z | +| host | Windows AMD64 | +| where | a local run (no workflow run URL) | +| driver | `shadow_compare_version` 2 | + +The adapter each leg executed, by digest — a path is not an identity, so a +stale build cannot stand in for the engine that was meant: + +| `own-shadow-engine` sha256 | bytes | +|---|---:| +| `94eda8fa31dd8c09443d60904d25c85b2670b584fc732bd787a8ea0458f5d9ef` | 1614336 | + +## The documents + +| document | target | mode | pin | raw bytes | raw sha256 | canonical sha256 | reduction | derived SARIF | declared-boundary | unexplained | wall clock (s) | timeout (s) | +|---|---|---|---|---:|---|---|---|---|---:|---:|---:|---:| +| `AvalonEdit.repo` | `AvalonEdit` | directory-walk | `ed0bd1490594` | 34507 | `f6e704f252f2` | `623af0f59c61` | identical | equal | 0 | 0 | 0.02 | 600 | +| `AvalonEdit.sln` | `AvalonEdit` | solution | `ed0bd1490594` | 34507 | `a5f2e5b70ff1` | `c3515fceb386` | identical | equal | 0 | 0 | 0.02 | 600 | +| `ClosedXML.repo` | `ClosedXML` | directory-walk | `4e89dcedd83c` | 201623 | `bbefbfb2089e` | `4599cfe379cd` | identical | equal | 0 | 0 | 0.11 | 600 | +| `ClosedXML.sln` | `ClosedXML` | solution | `4e89dcedd83c` | 201623 | `ec045c96b394` | `c6aabbe11592` | identical | equal | 0 | 0 | 0.11 | 600 | +| `MahApps.Metro.repo` | `MahApps.Metro` | directory-walk | `72099e310bac` | 44712 | `f6a22fecf1bb` | `a1f0d30bcad7` | identical | equal | 0 | 0 | 0.03 | 600 | +| `MahApps.Metro.sln` | `MahApps.Metro` | solution | `72099e310bac` | 44712 | `97f644b3b58a` | `d9ffd61f879b` | identical | equal | 0 | 0 | 0.02 | 600 | +| `MaterialDesignInXamlToolkit.repo` | `MaterialDesignInXamlToolkit` | directory-walk | `ef3a5ea434e3` | 53007 | `c0fa752cbe2d` | `3fd3416e502a` | identical | equal | 0 | 0 | 0.03 | 600 | +| `ShareX.repo` | `ShareX` | directory-walk | `0df9ca4d83ee` | 249448 | `8d94ea78b8ff` | `f3a30f10ac17` | identical | equal | 0 | 0 | 0.19 | 600 | +| `ShareX.sln` | `ShareX` | solution | `0df9ca4d83ee` | 249448 | `f01fa1d3d9ef` | `3a2c3c5eea1c` | identical | equal | 0 | 0 | 0.09 | 600 | +| `examples` | `examples` | directory-walk | `565de6d49f3d` | 13291 | `01e617d820d7` | `322523ad9073` | identical | equal | 0 | 0 | 0.02 | 600 | + +## The denominators, per target + +| target | extracted | compared | agreed | diverged | execution failures | input refusals | input disagreements | declared-boundary | unexplained | +|---|---:|---:|---:|---:|---:|---:|---:|---:|---:| +| `AvalonEdit` | 2 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | +| `ClosedXML` | 2 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | +| `MahApps.Metro` | 2 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | +| `MaterialDesignInXamlToolkit` | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | +| `ShareX` | 2 | 2 | 2 | 0 | 0 | 0 | 0 | 0 | 0 | +| `examples` | 1 | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | +| **total** | 10 | 10 | 10 | 0 | 0 | 0 | 0 | 0 | 0 | diff --git a/docs/notes/p022-bridge-verdict-final-acceptance.md b/docs/notes/p022-bridge-verdict-final-acceptance.md index c3392eec..9213c760 100644 --- a/docs/notes/p022-bridge-verdict-final-acceptance.md +++ b/docs/notes/p022-bridge-verdict-final-acceptance.md @@ -1,9 +1,10 @@ # P-022 step 6b (#259) — final acceptance: the coordinate-domain contract -> Status: **in execution.** final.0 the inventory (this note's §1–§2), final.1 -> the Python-first contract change, final.2 the Rust mirror and the promotion -> of the four boundary controls, final.3 the campaigns and the status surfaces. -> This note is both the plan and the completeness ledger. Counts are not typed +> Status: **final acceptance reached** (§7). The checkpoints it was executed +> in are on the record: final.0 the inventory (this note's §1–§2), final.1 the +> Python-first contract change, final.2 the Rust mirror and the promotion of the +> four boundary controls, final.3 the campaigns and the status surfaces. This +> note is both the plan and the completeness ledger. Counts are not typed > here — they live in the generated fragments > ([coordinate census](../generated/p022-coord-census.md), > [cp1 census](../generated/p022-cp1-census.md), diff --git a/docs/notes/p022-shadow-acceptance.md b/docs/notes/p022-shadow-acceptance.md index 66522ec7..18c3c3b6 100644 --- a/docs/notes/p022-shadow-acceptance.md +++ b/docs/notes/p022-shadow-acceptance.md @@ -3,10 +3,13 @@ > **Scope of this note.** It records the work that lands the owner decisions > ratified for #260's acceptance — D-4..D-7, B-2, B-3, R-1, R-2 in the > [owner-decision ledger](p022-shadow-infra-owner-decisions.md) — **over the -> committed corpus**. It is *not* #260's acceptance: the five-repository sweep -> and the large-solution controls are still owed, and nothing here may be read -> as shadow mode having been achieved, as P-022 being done, or as Rust being -> the default. +> committed corpus**. It was not, on its own, #260's acceptance: the +> five-repository sweep and the large-solution controls were owed, and they were +> taken afterwards and recorded in [the sweep +> note](p022-shadow-sweep.md) — which is where #260's final acceptance is +> claimed, and where the first two bullets of §6 below were discharged. Nothing +> in THIS note may be read as shadow mode having been achieved, as P-022 being +> done, or as Rust being the default. The infrastructure this builds on is checkpoints 1–4 (PR #338), and #259's final acceptance (PR #341) is what removed the last excuse for the verdict @@ -403,13 +406,19 @@ Six defects, none of them found by review, and each is a shape worth naming. ## §6 — Measured, not claimed * **The C# samples' OwnIR compares clean** — not measured here. This environment - has no .NET SDK; the gate is wired to take the measurement where one exists - (§2.5), and its result is reported rather than predicted. -* **The five-repository sweep and the large-solution controls** — not run. - #260's test matrix names them and they are separate work. -* **Windows and Linux path forms** — not exercised. The driver never sees a path - it did not read from, and the corpus is POSIX; the invariant it does prove is - byte-level and platform-independent, which is not the same claim. + had no .NET SDK; the gate is wired to take the measurement where one exists + (§2.5), and its result is reported rather than predicted. It has compared + clean on every pull request since. +* **The five-repository sweep and the large-solution controls** — not run *here*. + #260's test matrix names them and they were separate work; they have since + been run and recorded in [the sweep note](p022-shadow-sweep.md), whose §3 + names ten documents over six targets with their denominators. +* **Windows and Linux path forms** — not exercised *here*. The driver never sees + a path it did not read from, and the corpus is POSIX; the invariant this note + proves is byte-level and platform-independent, which is not the same claim. + The Windows leg was taken later, on a Windows host with an adapter built + there, and it found three defects in the driver and its controls before it + found agreement — [the sweep note's §5](p022-shadow-sweep.md). * **Rendered-byte parity of the three layer surfaces** — still each layer's own fixture family. The artifact carries layer outputs as JSON *values*, so a rendering difference there is invisible to this comparison. The **derived** @@ -425,8 +434,11 @@ Six defects, none of them found by review, and each is a shape worth naming. > Compare mode over the committed corpus: zero acceptance-unexplained at all > three layers and on the derived SARIF, on byte-attested same input, with the -> two OD-1 typed-door boundaries declared by policy. **Not #260's acceptance**: -> the five-repository sweep and the large-solution controls are still owed. +> two OD-1 typed-door boundaries declared by policy. **Not, on its own, #260's +> acceptance**: the five-repository sweep and the large-solution controls were +> owed, and they were taken separately — [the sweep +> note](p022-shadow-sweep.md) is where #260's final acceptance is claimed, and +> this note is the committed-corpus half beneath it. Not "shadow mode achieved". Not "P-022 done". Not "Rust is the default", which is #261/#262's cutover and is untouched — Python remains the public engine, no diff --git a/docs/notes/p022-shadow-sweep.md b/docs/notes/p022-shadow-sweep.md new file mode 100644 index 00000000..58820643 --- /dev/null +++ b/docs/notes/p022-shadow-sweep.md @@ -0,0 +1,569 @@ +# P-022 step 7a (#260) — the sweep: five pinned repositories, large-solution controls, examples, path forms + +> **Scope of this note.** It is the record of #260's **final acceptance +> measurement** — the part of its test matrix the acceptance surfaces over the +> committed corpus deliberately did not take: the five pinned OSS repositories +> of #243 at their pinned commits, the large/multi-project solution controls, +> the `examples/` tree, and the Windows path forms. The contract it runs under +> is D-4..D-7, B-2, B-3, R-1 and R-2 in the +> [owner-decision ledger](p022-shadow-infra-owner-decisions.md); this note may +> not reopen any of them. What the committed corpus already proves is +> [the acceptance note](p022-shadow-acceptance.md); this is its §6 first two +> bullets being taken rather than restated. + +**Where the numbers are.** Every count this sweep *produced* — documents, +outcomes, denominators per target, byte sizes, wall-clock — lives in +[`docs/generated/p022-shadow-sweep.md`](../generated/p022-shadow-sweep.md), +rendered by `scripts/render_checkpoint_status.py` from the recorded run, and is +reached from here by link. The inventory below is the other kind of number: the +*inputs* measured from the checkouts before anything ran, so that the matrix +this note promises is a matrix somebody measured rather than one somebody +estimated. Those are stated here and also carried as data in +[`docs/evidence/p022-shadow-sweep.json`](../evidence/p022-shadow-sweep.json). + +--- + +## §1 — The targets, the pins, and what this environment can actually do + +### 1.1 The five, and the pin rule + +The targets and commits are #243's, **reused verbatim** and never re-resolved +against upstream HEAD. They are copied from +[`docs/notes/precision-remeasure-2026-07-12-data/`](precision-remeasure-2026-07-12-data/), +which is the machine-readable record of the last sweep that used them, not from +the prose of the note beside it. + +| target | upstream | pinned commit | +|---|---|---| +| ShareX | `ShareX/ShareX` | `0df9ca4d83eed9d2489048c539d7d1fc2860fdec` | +| MahApps.Metro | `MahApps/MahApps.Metro` | `72099e310bac2d12ac98fd7560b69679252519f5` | +| MaterialDesignInXamlToolkit | `MaterialDesignInXAML/MaterialDesignInXamlToolkit` | `ef3a5ea434e39182b1848f5e11aaea6b3890581f` | +| AvalonEdit | `icsharpcode/AvalonEdit` | `ed0bd149059469ac9bd39b13cf8a341b12a6c1da` | +| ClosedXML | `ClosedXML/ClosedXML` | `4e89dcedd83cad553e84d2d97f77fc3d7deb630f` | + +**The pin rule.** A checkout is materialized by fetching *the commit itself* +(`git fetch --depth 1 origin `, then `git checkout --detach FETCH_HEAD`), +so what lands is the pin rather than a branch tip that happens to contain it. +Before anything else runs, `git rev-parse HEAD` is compared with the pin. **A +mismatch is a hard failure of that target, not a warning and not a newer +measurement.** A target that moved has not been measured; it has been replaced. + +The same rule applies to the *port's* half. The adapter is named in every +recorded comparison by `sha256` and byte length as well as by path, and +`OWN_SHADOW_ENGINE` is always set explicitly — a driver that found +`rust/target/debug/own-shadow-engine` because the environment was unset +compared *an* engine, not *the* engine. + +### 1.2 What this environment is + +This environment **can** run the whole matrix, so it does: the G.0 fallback +(take the measurement through the workflow and read its artifacts back) is not +used, and the recorded run is a local one whose commands, commits and artifact +identities are on the record. What made that possible, measured rather than +assumed: + +| capability | what is present | +|---|---| +| .NET SDK | 10.0.400 — builds the extractor's pinned `net8.0` target | +| .NET runtimes | `Microsoft.NETCore.App` 8.0.30 and `Microsoft.WindowsDesktop.App` 8.0.30 | +| Roslyn | pinned by the extractor itself (`Microsoft.CodeAnalysis.CSharp` 4.9.2), so the SDK version does not move the parse | +| Rust | 1.98.1 (`x86_64-pc-windows-msvc`), installed for this work; MSVC 14.44 + Windows SDK 10.0.26100 were already present | +| host | Windows 11 — which is why the path-form leg of the matrix is a *local* measurement here and a `windows-latest` job in CI | + +The reference pack is materialized exactly the way `ci.yml`'s +`corpus-benchmark` job does it — a scratch `net8.0-windows` project with +`UseWPF` / `UseWindowsForms` / `EnableWindowsTargeting`, `dotnet restore`, then +the resolved `ref/net8.0` directory exported as `OWN_EXTRA_REF_DIRS`. It +resolves to `microsoft.windowsdesktop.app.ref` **8.0.30** here; the 2026-07-12 +remeasure resolved **8.0.28**. Both carry 47 DLLs. That difference is recorded +rather than pinned away because **it cannot bias this measurement**: whatever it +does to a facts document, both engines receive that document as the same +bytes. It would matter to a precision remeasure; this is not one. + +### 1.3 The disk and wall-clock the matrix costs + +Measured on this machine before the matrix was promised, one target extracted +first (AvalonEdit, the smallest) and then the rest: + +| item | measured | +|---|---| +| the five checkouts, on disk | ~147 MB total (AvalonEdit ~3 MB, ClosedXML ~26 MB, MahApps.Metro ~32 MB, ShareX ~41 MB, MaterialDesign ~45 MB) | +| extraction, all ten documents | ~98 s total; the slowest single document is ShareX's directory walk at ~20 s | +| the ten facts documents | ~1.1 MB total; the largest is ~250 KB | +| compare, per document | well under a second — the committed-corpus gate runs 104 documents in ~0.7 s | + +The consequence for the timeout policy is in §2.4: nothing here is near the +driver's default, and the default is therefore kept and *recorded* rather than +raised on a guess. + +--- + +## §2 — The matrix, exactly as it will be run + +### 2.1 Which targets carry a solution — measured, from the checkout + +Counted by resolving each `.sln`'s `Project(...)` entries to `.csproj` members +that exist on disk, and counting the `.cs` files under those members: + +| target | solutions found | member projects | `.cs` under members | +|---|---|---|---| +| ShareX | `ShareX.sln` | 13 | 1126 | +| | `ShareX.ImageEditor.sln` | 2 | 433 | +| ClosedXML | `ClosedXML.sln` | 6 | 921 | +| MahApps.Metro | `src/MahApps.Metro.sln` | 4 | 328 | +| AvalonEdit | `ICSharpCode.AvalonEdit.sln` | 3 | 248 | +| | `ICSharpCode.AvalonEdit.Documentation.sln` | 1 | 214 | +| MaterialDesignInXamlToolkit | **none** — `MaterialDesignToolkit.Full.slnx` only | — | — | + +**MaterialDesignInXamlToolkit has no classic solution at its pinned commit.** +It carries the XML solution format (`.slnx`), and the extractor's solution +resolver reads the classic `Project("{...}") = "name", "path"` form only. That +is a *measured gap*, recorded here and nowhere else acted on: teaching the +extractor `.slnx` would be a production change, and this task changes no +production behaviour. The target is therefore covered by its directory walk +alone, and the record says so per target rather than averaging it away. + +### 2.2 The large-solution controls taken + +#260 asks for "selected large/multi-project solutions" and this task's brief for +"at least the two largest". Taken: **the largest solution of every target that +has one** — `ShareX.sln` (13 members), `ClosedXML.sln` (6), `MahApps.Metro.sln` +(4), `ICSharpCode.AvalonEdit.sln` (3). The two largest are the first two; the +other two are nearly free and make the mode's coverage per-target rather than +per-sample. `ShareX.ImageEditor.sln` and the AvalonEdit documentation solution +are proper subsets of the solutions already taken and are skipped for that +reason, which is a choice on the record rather than a silent omission. + +The solution documents are **not** the directory-walk documents under another +name. Measured on ShareX: the two documents carry the same 106 components and +74 functions, in a **different order** — the fan-out enumerates the source set +project by project, the walk enumerates it in directory order — so they have +identical byte length and different digests, and they are two documents, not +one. That is exactly the property a large-solution control is for: a different +extractor path, a differently ordered document, and per-layer ordering +semantics that are *declared* rather than normalized away. + +### 2.3 The ten documents, and the commands, verbatim + +Ten documents, each extracted **exactly once**. `scripts/own-check.sh` runs the +extractor once into a temporary file and `--emit-facts ` persists exactly +that file; stage 2 (`python -m ownlang ownir`) is the reference's *verdict* path +and is neither a second extraction nor the comparison. The comparison is the +driver's, taken from the emitted bytes. + +```bash +# per document, with the materialized WindowsDesktop ref pack (§1.2) +OWN_EXTRA_REF_DIRS= scripts/own-check.sh \ + --format sarif --severity warning \ + --emit-facts -- +``` + +| document | mode | `` | +|---|---|---| +| `ShareX.repo` | directory walk | `/ShareX` | +| `MahApps.Metro.repo` | directory walk | `/MahApps.Metro` | +| `MaterialDesignInXamlToolkit.repo` | directory walk | `/MaterialDesignInXamlToolkit` | +| `AvalonEdit.repo` | directory walk | `/AvalonEdit` | +| `ClosedXML.repo` | directory walk | `/ClosedXML` | +| `ShareX.sln` | `.sln` fan-out | `/ShareX/ShareX.sln` | +| `ClosedXML.sln` | `.sln` fan-out | `/ClosedXML/ClosedXML.sln` | +| `MahApps.Metro.sln` | `.sln` fan-out | `/MahApps.Metro/src/MahApps.Metro.sln` | +| `AvalonEdit.sln` | `.sln` fan-out | `/AvalonEdit/ICSharpCode.AvalonEdit.sln` | +| `examples` | directory walk | `/examples` | + +The driver then runs **once** over a manifest naming all ten: + +```bash +cd rust && cargo build --release -p own-shadow --bin own-shadow-engine +OWN_SHADOW_ENGINE= \ + python scripts/shadow_compare.py --engine compare \ + --manifest --out --quiet +``` + +and the Windows path-form leg is the committed-corpus gate, on this host, with +the adapter built on this host: + +```bash +OWN_SHADOW_ENGINE= \ + python scripts/shadow_compare.py --engine compare --corpus --quiet +OWN_SHADOW_ENGINE= OWN_SHADOW_COMPARE_REQUIRED=1 \ + python tests/test_shadow_compare.py +``` + +### 2.4 What "covered" means here, and the timeout policy + +**A repository is not covered because extraction succeeded, and a solution is +not covered because some project inside it emitted OwnIR.** Coverage is a +recorded, non-empty set of OwnIR documents fed byte-identically to both engines +and judged by compare mode — *per target, with the denominator on the record*. +This is the direct descendant of the fifth failure mode in #250: a green gate +over an empty set is worse than a red one, because a red one at least says it +is awake. Hence three rules the driver enforces rather than this note asserting +them: + +* a run that compared **zero** documents is a **failure**, never agreement; +* a **target** whose compare-attempted count is zero is a **failed target**, + never a passed repository; +* every document's `facts_sha256` in the manifest is checked against the bytes + the driver actually read, and every document is validated **before any engine + runs**. + +**Timeouts.** Explicit per document and recorded per document. §1.3 measured the +whole matrix as sub-second per compare, three orders of magnitude inside the +driver's 120 s default, so the default is what the manifest carries — stated as +a number in the manifest rather than inherited silently, so that raising it for +a future target is a visible edit. A timeout is an **execution failure with a +report** (R-2), never a silent skip. + +### 2.5 The sweep ledger schema + +Two documents, the same split every campaign in this tree uses — a +**definition** that says what should be measured and a **result** that records +one actual run of it, with one interpreter reading both. + +`docs/evidence/p022-shadow-sweep.json` — the definition: + +```text +schema, comment, sweep +driver_version the shadow_compare_version this definition expects +default_timeout_seconds the default a manifest entry inherits +adapter_build_command how the port's half is produced +driver_command how the run is taken +reference_pack, pin_rule the environment and the drift rule, as prose +targets[] { target, upstream, pinned_commit, solutions[], slnx[], note } +documents[] { id, target, target_commit, extraction_mode, + extraction_command, timeout_seconds } +``` + +`pinned_commit` and a document's `target_commit` are **null** for the one target +whose pin is this repository itself (`examples`): the interpreter then requires +the recorded pin to equal the run's own `source_commit`, which is a check rather +than an exemption. + +The **manifest** the driver is actually given is a third document, not +committed: it is the definition's rows joined to the facts files a run produced, +each with the `facts_sha256` of the bytes on disk and the `id` that ties it back +to the definition. It is not committed because it names paths on the machine +that ran, and because regenerating it is how a re-run is taken. + +`docs/evidence/p022-shadow-sweep.result.json` — one run: + +```text +schema, sweep, definition, definition_sha256 +source_commit, recorded_at, host, workflow_run_url (null for a local run) +adapters[] { sha256, bytes } (one per leg; CI legs build their own) +driver_version +documents[] { id, source, target, target_commit, extraction_mode, + extraction_command, facts_sha256, raw { digest, bytes }, + canonical { digest, bytes }, outcome, timeout_seconds, + reduction_outcome, by_kind{}, by_acceptance{}, + derived_outcome, wall_clock_seconds } +targets[] { target, documents_extracted, compare_attempted, agreed, + diverged, execution_failures, input_refusals, + input_disagreements, declared_boundary_observations, + acceptance_unexplained_observations } +totals { the same fields, summed } +``` + +`tests/shadow_sweep.py` is the single interpreter of the pair. +`scripts/render_checkpoint_status.py` renders +`docs/generated/p022-shadow-sweep.md` from it and +`tests/test_checkpoint_status.py` gates that the committed fragment equals the +projection — the same pipeline every campaign count in this repository already +goes through. **A re-run replaces the result whole; it is never patched.** + +### 2.6 The churn budget, written before anything moved + +* `tests/fixtures/` — **nothing**, unless a divergence produces an artifact + small enough to be a control, in which case it is added insertion-stable and + the finding that produced it is on the record with it. The frozen goldens do + not move for a measurement. +* `docs/evidence/` — gains the sweep definition and one recorded result, plus + one campaign definition and result over the driver's new pieces. +* `docs/generated/` — gains `p022-shadow-sweep.md` and the campaign fragment + regenerates. +* **The five repositories' facts documents do not enter the repository.** They + are large and exactly reproducible from the pins and the commands in §2.3; + their *identities* (the raw digest and byte length of each, and the canonical + identity both engines derived) are what the result records. +* **Production behaviour changes nowhere.** Not in `ownlang/`, not in the + analyses, not in the renderers, not in `own-check.sh`'s verdict path. The + driver, the adapter, the workflow and the evidence pipeline are dev tooling. +* **The comparison machinery is frozen for the measurement.** The canonical + form, the artifact verifier, the trace, the reducer, `BOUNDARY_POLICY`, the + derived-SARIF configuration and the driver's *judgement* do not move while + the sweep runs. A divergence is a finding first; adjusting a normalizer to + make one go away is a contract change that waits for the owner. + +--- + +## §3 — What ran, and what it found + +One run, taken locally at `565de6d`, recorded whole in +[`docs/evidence/p022-shadow-sweep.result.json`](../evidence/p022-shadow-sweep.result.json) +and interpreted into +[`docs/generated/p022-shadow-sweep.md`](../generated/p022-shadow-sweep.md), +which is where every count of it lives. Per **document** it names the pin that +was verified, the extraction mode, the raw and canonical identities and the byte +size, the outcome, the derived-SARIF outcome, the wall clock and the timeout +that was in force. Per **target** it names the denominators: documents +extracted, documents compared, and the outcome breakdown — so a target that had +been skipped would appear at zero rather than not appear at all. + +**The environment, exactly as §1.2 promised and with the two differences from +the 2026-07-12 remeasure on the record.** The extractor is the pinned +`Microsoft.CodeAnalysis.CSharp` 4.9.2 built by a .NET 10.0.400 SDK against the +8.0.30 runtime (the remeasure used an 8.0.422 SDK), and the reference pack +resolved to `microsoft.windowsdesktop.app.ref` **8.0.30** (the remeasure +resolved 8.0.28) — 47 DLLs either way. Neither can bias this measurement: +whatever they do to a facts document, both engines receive that document as the +same bytes. They would matter to a precision remeasure, and this is not one. + +**Extraction is reproducible, and that was checked rather than assumed.** The +ten documents were extracted three times on this host — for §1.3's wall-clock +inventory, and twice more as the recorded run was re-taken at a later commit — +and all ten digests are identical across all three passes. + +The commands are §2.3's, verbatim, and they are carried as data in the sweep +definition so the workflow, the note and the record cannot drift into three +readings of one command. + +**The campaigns.** Three are recorded on a clean tree, each at the commit it +measured, with no survivor and no missed catcher: the new `p022-shadow-sweep-1` over the driver's +version-2 surfaces and the sweep interpreter, and `p022-shadow-acc-1` and +`p022-shadow-acc-2` re-run because this branch moved both their target and their +catcher files — which is the re-run rule the acceptance note wrote down. Two of +`acc-2`'s mutations were **re-anchored** onto the moved compare path and each +still expresses its own rule: "the driver stops failing the run when an engine +crashes" and "the driver hands the port bytes it did not hand the reference". +Every count is in [the campaign +fragment](../generated/p022-shadow-mutations.md). + +## §4 — The differential this run asserts + +The gate is a property of a green run, not a number in this note. + +* **Every document in the definition is in the result**, measured at the pin, + in the mode and by the command the definition declares — `tests/shadow_sweep.py` + refuses a run that is short, a run that is long, and a run that measured + something else. +* **Every document's outcome is `agreed`**, and no observation is + acceptance-unexplained — including inside a document recorded as agreed, + which is checked separately because an outcome that contradicts its own + observations is a finding rather than a rounding. +* **Every target's compare-attempted count equals its documents-extracted + count, and neither is zero.** +* **The totals are the sum of the per-target rows**, recomputed rather than + read. +* **The run names the engine it ran** by `sha256` and byte length, and the + driver version the definition expects. +* **The run's `source_commit` is an ancestor of HEAD** — the same provenance + rule every recorded campaign in this tree is held to. + +Any one of those failing is a red `tests/run_tests.py`, because the fragment is +rendered from the same interpreter the gate runs. + +## §5 — Findings + +**No document diverged, so G.4's three resolution paths were not needed: there +is no port bug, no reference question and no new boundary here.** The frozen +comparison machinery — the canonical form, the artifact verifier, the trace, +the reducer, `BOUNDARY_POLICY`, the derived-SARIF configuration and the +driver's judgement — was not touched, and no fixture under `tests/fixtures/` +moved. + +What the measurement did find is six defects in the *harnesses*, all six +invisible on the platform CI runs on, and each is a shape worth naming. Three of +them are in the mutation harness, and they compound: together they meant that no +campaign in this repository could be recorded anywhere but Linux, and that the +one which failed loudest failed only after the other two had been fixed. + +1. **The driver died on a label.** `os.path.relpath` raises `ValueError` for a + path on another Windows drive, and the sweep's facts documents live outside + the checkout by design — so the very first sweep document killed the driver + before either engine ran, over the string a result records as its `source`. + The fix is a label that falls back to the absolute path; the shape is worth + remembering, because the failing code was the one line in the loop that had + nothing to do with comparing anything. + +2. **The timeout was not a timeout.** `subprocess.run(timeout=…)` kills the + child and then waits for the pipes to close, so a surviving grandchild + blocks the driver for ever. Owner decision R-2 makes a timeout a run-level + hard failure the driver has to *report*, which a driver that never returns + does not do. The adapter starts nothing, which is exactly why this had never + been hit — the property is about what the driver guarantees, not about what + today's adapter happens to do. The child now gets its own process group on + POSIX and is killed as a tree on Windows. + +3. **The control group that would have caught it could not run at all.** The + driver invokes its adapter as one argv entry (R-1: no arguments), and + Windows cannot start a `.py` — `CreateProcess` does not consult file + associations. Every double-driven control failed there with `WinError 193`: + the group whose stated purpose is that it "runs everywhere, including the + Python-only test matrix" ran nowhere on Windows, and had done since it was + written. A launcher beside the double fixes it without touching the driver's + contract. Finding 2 was found five minutes later, by the control that could + finally execute. + +4. **The mutation harness refused its own work and blamed the tree.** + `run_campaign` restores each target and then compares `git status` with the + baseline; it read and wrote through Python's text mode, so on a CRLF working + copy every campaign rewrote its targets' line endings and died with "the + working tree changed during S01 — the run is void". The tree had changed. + What changed it was the harness, and no campaign could be recorded on that + platform at all. Its own "was it restored?" check never saw it, because that + check compares *text*: it read the rewritten file back through the same + translation and got the same string. A property about bytes on disk is not + observable from above them, which is why the control drives `read_source` / + `write_source` directly. + +5. **A catcher's name took the host's spelling, and under-reported rather + than failed.** cargo prints its test target with the host separator, so on + Windows a failing test is recorded as `own-shadow/tests\repro.rs::…` where + every campaign definition and every committed result says + `own-shadow/tests/repro.rs::…`. Measured on a real re-run of + `p022-shadow-acc-1`: five of twelve mutations reported "expected catchers + MISSED" while the catcher list printed beside them named exactly the test + that had been expected. That is worse than an outright failure, because the + answer looks like a finding — "a rule went unprotected" — and would have been + filed as one. A catcher name is an identity; the target is normalized where + it is parsed, and the control drives the parser with both spellings of one + cargo run. + +6. **A layer's output was decoded with the console codepage.** `_run_layer` + read its child with `text=True` and no encoding, so Python used the locale + encoding; on a cp1251 console one non-ASCII byte anywhere in cargo's output + raised `UnicodeDecodeError` inside `communicate()` and took the campaign with + it, mid-run, after a production file had already been mutated. The + finally-block restored the tree, so nothing was left broken — what was lost + was the run. cargo emits UTF-8 and this repository's own test names contain + an em dash, so the byte was never going to be exotic. Both subprocess calls + now decode UTF-8 explicitly with `errors="replace"`: a stray byte costs one + character rather than the campaign. + +Two measured facts about the matrix itself, neither of them a defect: + +7. **MaterialDesignInXamlToolkit has no classic solution at its pin.** It + carries `MaterialDesignToolkit.Full.slnx`, and the extractor's solution + resolver reads the classic `Project("{…}") = "name", "path"` form only. + Teaching it `.slnx` would be a production change, which this task does not + make; the target is covered by its directory walk alone, and the record says + so per target rather than averaging it away. + +8. **A solution document is not a directory-walk document under another name, + and it is not a different document in the way one might expect either.** + Measured on ShareX: the two carry the same components and functions in a + different ORDER — the fan-out enumerates project by project, the walk in + directory order — so they have equal byte length and different digests. That + is a better control than a subset would have been: the two engines are + compared on a reordered document, and per-layer ordering semantics are + *declared* rather than normalized away. + +## §6 — Measured, not claimed + +* **Windows path forms.** Measured, on this host, with an adapter built here: + the committed-corpus gate compares clean over every committed document, the + driver's own controls run and pass under `OWN_SHADOW_COMPARE_REQUIRED=1`, and + the sweep's own ten documents were compared on the same host with paths on a + different drive from the checkout. What is **not** claimed: anything about + GitHub's `windows-latest` runner. The job is wired in + `.github/workflows/shadow-sweep.yml` and has not run, because a + `workflow_dispatch` workflow is only dispatchable once it is on the default + branch. +* **Linux path forms.** Not exercised by this run. They are exercised by the + fast gate that already exists — `shadow compare (committed corpus)` and + `shadow compare (C# samples)` are `ubuntu-latest` jobs in `ci.yml` — and the + sweep workflow's own legs are `ubuntu-latest` too, unrun for the reason + above. +* **The C# samples.** Not re-measured here: that leg is a CI job fed by the + `wpf-extractor` job's own OwnIR through one upload-artifact handshake, and it + compared clean on `main` when PR #342 landed it. This branch's CI re-measures + it; nothing in this note stands in for that result. +* **The sweep workflow itself.** Written and committed, never executed. The + recorded run is a **local** one, which is what #260 allows ("the + five-repository sweep may use a local/corpus-capable agent, but its commands, + commits and output artifacts must be recorded"), and the record carries + `workflow_run_url: null` rather than pretending otherwise. +* **Precision.** This sweep does not re-measure it. The finding counts the + extractor produced over the five targets are not compared with #243's, and + they would not be comparable: many analyses have landed since, and two + environment components differ (§3). What is compared is one engine against + the other over one byte sequence. +* **The adapter digest names the BUILD, not the port's source**, and that was + measured on purpose rather than inferred: `cargo clean --release -p + own-shadow` followed by the same release build, over a `rust/` tree that did + not change by one byte, produced a different digest at exactly the same byte + length (1 614 336). That is what the #342 review asked for — "so a stale build + can never stand in for the engine that was meant" — and it is emphatically not + a content hash of the port: two legs of one sweep on two runners will name two + digests, which is why the record carries the SET of adapters a run executed + rather than asserting one. +* **The five repositories' facts documents are not in the repository.** Their + identities are: the raw digest and byte length of each, and the canonical + identity both engines derived from it, are in the recorded run. They are + exactly reproducible from the pins and §2.3's commands. +* **Rendered-byte parity of the three layer surfaces** stays each layer's own + fixture family, as the acceptance note's §6 already records. The artifact + carries layer outputs as JSON *values*; the derived SARIF surface is the one + rendered surface compared byte-exactly, and it was compared on all ten + documents here. + +## §7 — The wording this earns + +> **#260 final acceptance reached: dual-engine compare mode reports zero +> acceptance-unexplained over its full test matrix — the committed corpus, the +> C# samples, the examples, the five pinned OSS repositories and the +> large-solution controls — at all three layers and on the derived SARIF, on +> byte-attested same input, with the OD-1 typed-door boundaries declared by +> policy; Python remains the public engine.** + +What that sentence rests on, leg by leg, so that it can be checked rather than +believed: the five repositories, the four large-solution controls and +`examples/` are the recorded run in §3, taken locally and gated by +`tests/shadow_sweep.py`; the committed corpus is the `ci.yml` job that has +gated every pull request since PR #342, re-measured here on Windows as well; +the C# samples are the `ci.yml` job fed by the extractor's own OwnIR, which +this branch's CI re-measures (§6). The OD-1 boundaries are the two typed-door +documents of the committed corpus and are unchanged; the sweep's own ten +documents produced **no** boundary observations at all, because nothing in them +reaches that door. + +Not "P-022 done". Not "Rust is the default", which is #262's cutover behind +#261 — Python remains the public engine, `own-shadow-engine` is a dev-only +adapter with no command surface to grow out of, and no production behaviour +changed anywhere in this work. Not "parity" as a bare word: what was measured +is compare mode's judgement over a named set of documents, and the set is on +the record with its denominators. + +### The commands this note's claims come from + +```text +# the adapter, and the sweep over the ten documents of the definition +cd rust && cargo build --release -p own-shadow --bin own-shadow-engine +OWN_EXTRA_REF_DIRS= scripts/own-check.sh \ + --format sarif --severity warning --emit-facts -- # per document +OWN_SHADOW_ENGINE= \ + python scripts/shadow_compare.py --engine compare --manifest \ + --out +python tests/shadow_sweep.py --collect \ + --write docs/evidence/p022-shadow-sweep.result.json +python tests/shadow_sweep.py +python scripts/render_checkpoint_status.py + +# the Windows path-form leg +OWN_SHADOW_ENGINE= \ + python scripts/shadow_compare.py --engine compare --corpus --quiet +OWN_SHADOW_ENGINE= OWN_SHADOW_COMPARE_REQUIRED=1 \ + python tests/test_shadow_compare.py + +# the sweep in CI, once this workflow is on the default branch +gh workflow run "shadow sweep (#260)" + +# the campaigns re-run in this branch, and the new one +python scripts/mutate_campaign.py --campaign docs/evidence/.json --validate +python scripts/mutate_campaign.py --campaign docs/evidence/.json --run + +python tests/run_tests.py +ruff check . && mypy +cd rust && cargo test --workspace --no-fail-fast +``` diff --git a/docs/proposals/P-022-rust-core-migration.md b/docs/proposals/P-022-rust-core-migration.md index 6ab063e5..857db0e4 100644 --- a/docs/proposals/P-022-rust-core-migration.md +++ b/docs/proposals/P-022-rust-core-migration.md @@ -74,25 +74,28 @@ was #258 alone, which is satisfied. Per the checkpoints #259 itself defines: | 5c | `own-codegen` (analysis-independent sibling) | #257 | **ready**, independent of the analysis path — parallelizable | | 6a | OwnIR **bridge semantics formalized** before the port | #258 | **complete** — see above | | 6b | Rust `own-bridge`, layered OwnIR parity | #259 | **final acceptance reached** — see the checkpoint table and the line above it | -| 7a | dual-engine shadow mode + zero-diff reproduction artifacts | #260 (supported by #269) | **infrastructure COMPLETE (checkpoints 1–4); the acceptance decisions are LANDED over the committed corpus; final acceptance still owes the sweep**. What the acceptance work measures, and the only wording it earns: compare mode over the committed corpus reports **zero acceptance-unexplained at all three layers and on the derived SARIF**, on byte-attested same input, with the two #294 OD-1 typed-door boundaries declared by policy. It is **not** #260's acceptance, not shadow mode achieved, and not "Rust is the default" (#261/#262's cutover): #260's test matrix also names a five-repository sweep and large-solution controls, and neither is taken. The owner decisions this lands are recorded verbatim in [the ledger](../notes/p022-shadow-infra-owner-decisions.md) — **D-4** the reduction scope IS the layer order (aliased, never a third copy); **D-5** observation *kind* and *acceptance* as orthogonal fields, with a frozen `(layer, kind, class)` boundary policy the refusing engine declares structurally and the reducer matches by triple, never by text; **D-6** canonical SARIF as a DERIVED surface rather than a layer, under one named render configuration; **D-7** the BR-V8 address as a pairing address whose duplicate ordinal is part of it; **B-2** artifact v3 carrying the raw bytes and each engine's `consumed`; **B-3** every `consumed` from a real run, no promotion; **R-1** a dev-only `own-shadow-engine` adapter (stdin bytes, stdout one capture, no paths — no #261 cutover surface); **R-2** a crash, timeout or non-zero exit as a run-level hard failure that is never a refusal and never a fallback. B-1's byte-level invariant, which the infrastructure slice could not prove, is now proved: the artifact carries `input.raw` and every engine entry a `consumed` taken before any decode, and verification walks the whole chain. The checkpoints stand as they landed: cp1 same-input capture + the artifact format; cp2 the engine protocol; cp3 the `AnalysisTrace` (#269); cp4 first-divergence reduction. Fourteen mutation campaigns are recorded at one commit with no survivors and no missed catchers, and they found six defects review did not — among them a control that could not tell two enforcement points apart, a measurement that measured its own harness, and a mutation recorded as CAUGHT by a rule it never expressed. Every count lives in the generated fragments ([census](../generated/p022-shadow-census.md), [campaigns](../generated/p022-shadow-mutations.md)) and never here; the record is [the acceptance note](../notes/p022-shadow-acceptance.md), which also names what is measured-not-claimed. Python remains the public engine and no production behaviour changed | +| 7a | dual-engine shadow mode + zero-diff reproduction artifacts | #260 (supported by #269) | **final acceptance REACHED**. The only wording it earns: *dual-engine compare mode reports zero acceptance-unexplained over its full test matrix — the committed corpus, the C# samples, the examples, the five pinned OSS repositories of #243 and the large-solution controls — at all three layers and on the derived SARIF, on byte-attested same input, with the OD-1 typed-door boundaries declared by policy; Python remains the public engine.* It is **not** "P-022 done" and **not** "Rust is the default", which is #262's cutover behind #261. The sweep is what the acceptance surfaces over the committed corpus deliberately left owed: ten documents over six targets, each repository at its **verified** pin (drift is a failed target, never a newer measurement), each extracted **once** through `own-check.sh --emit-facts` and compared from those bytes — the five directory walks, the largest `.sln` of every target that has one (a different extractor path, and measurably a differently *ordered* document rather than a subset), and `examples/`. Coverage is defined so that it cannot be faked: a repository is not covered because extraction succeeded, so the driver fails a run that compared zero documents AND a declared target it never reached, and the **denominators are recorded per target**. The driver gained the identity the #342 review asked for — every result and failure report names the adapter by `sha256` and byte length, taken from the file that ran — plus manifest runs whose every document is verified against its `facts_sha256` before any engine starts (`shadow_compare_version` 2; the artifact format v3 is untouched). Taking the measurement found six HARNESS defects and no engine divergence: a cross-drive `relpath` that killed the driver on a label, a timeout that never returned when the adapter had children, a control group that could not execute on Windows at all (and so had never caught the timeout one), and three in the mutation harness that between them meant no campaign could be recorded anywhere but Linux — rewritten line endings that made it refuse its own run, a catcher name that took the host's path separator and so reported five protected rules as unprotected, and a layer decoded with the console codepage. The five repositories' facts documents are not committed — their identities are. The scheduled/manual gate is `.github/workflows/shadow-sweep.yml`; every count lives in the generated fragments ([sweep](../generated/p022-shadow-sweep.md), [census](../generated/p022-shadow-census.md), [campaigns](../generated/p022-shadow-mutations.md)) and never here; the records are [the sweep note](../notes/p022-shadow-sweep.md) and [the acceptance note](../notes/p022-shadow-acceptance.md), which name what is measured-not-claimed. The owner decisions remain D-4..D-7, B-2, B-3, R-1 and R-2 in [the ledger](../notes/p022-shadow-infra-owner-decisions.md), unreopened. No production behaviour changed | | 7b | Rust `own-cli`: command/output/exit-code parity | #261 | blocked — needs the production bridge and the output surfaces | | 8 | Rust-default **cutover**, rollback gate, Python distribution removal | #262 | blocked by #260/#261 and final parity | -**Preferred queue:** #260's sweep → #261. cp5, 4b, the coordinate-domain +**Preferred queue:** #261. cp5, 4b, the coordinate-domain decision and #260's own acceptance decisions are all off this queue: each is complete at its surface. The #260/#269 *infrastructure* slice ran in parallel by design — it is measured on the landed checkpoints and takes no position on the ones that are open, which is why its first checkpoint could land without waiting on cp5. -What #260's **final acceptance** now waits on is a measurement rather than a -decision, and that is the change: its two blocking decisions are ratified and +#260's **final acceptance is reached**, and the last thing it waited on was a +measurement rather than a decision: its two blocking decisions were ratified and landed (D-4..D-7, B-2, B-3, R-1, R-2 in [the -ledger](../notes/p022-shadow-infra-owner-decisions.md)), compare mode runs over -the committed corpus in CI, and what remains is the five-repository sweep and -the large-solution controls #260's test matrix also names. Neither is taken, so -the row above says "compare mode over the committed corpus" and not "shadow -mode". +ledger](../notes/p022-shadow-infra-owner-decisions.md)), compare mode has run +over the committed corpus in CI since PR #342, and the five-repository sweep and +the large-solution controls its test matrix also names are now taken and +recorded ([the sweep note](../notes/p022-shadow-sweep.md), [the generated +record](../generated/p022-shadow-sweep.md)). The row above therefore says +"compare mode over its full test matrix" — and still does not say "shadow mode +achieved", "P-022 done" or "Rust is the default", none of which follows from a +comparison being clean. The coordinate DOMAIN that used to head this queue landed the same way and for the same reason (see the cp5 row and the diff --git a/docs/proposals/README.md b/docs/proposals/README.md index 999d4019..10c461a5 100644 --- a/docs/proposals/README.md +++ b/docs/proposals/README.md @@ -41,7 +41,7 @@ proposal is marked `done` with a pointer. | [P-017](P-017-multi-stack-frontends.md) | Multi-stack frontends (OwnTS / OwnJVM: OwnJava + OwnKotlin) | draft | | [P-020](P-020-ownts-react-effects.md) | OwnTS React effects profile (`Own.React`) — the effect-storm angle | draft | | [P-021](P-021-async-audit-pack.md) | Async audit pack (`Own.Async`) | draft | -| [P-022](P-022-rust-core-migration.md) | Rust core migration: crate DAG, patterns, prior art, differential oracle (Python = golden) | in execution — steps 0–4 built (#214/#249); step 5a done (full diagnostic contract, #255 via #319/#320/#321); step 5b SARIF done (#256; `.ownreport.json` struck — a buffer report needing the AST, not a diagnostics surface); step 6a done (`spec/Bridge.md`, #258); step 6b complete at final acceptance (`own-lowered`/`own-bridge`, #259: lowering and MOS parity landed; strict-door validation complete with no known divergence — the first 0/0/0 proved to be the ledger agreeing with its own author, and the second omitted two families that a Python-first defensive-limit change (#326) had to close before the third could measure them; analysis wiring complete at the checkpoint-4 surface — `check_facts` through the real analyses, Layer 3 goldens built, with an executable exclusion ledger naming each declared boundary; **cp5 complete at its surface** — the replay compares EVERY `Finding` member (the BR-V4 wording matrix and the BR-V5 evidence slices included) and every refusal in full, and a second fixture family freezes the BR-V9 rendered surfaces byte for byte, all against goldens none of which was regenerated; **row 4b complete** — the obligation-protocol analysis (OBL001–005) is ported into `own-analysis`, its typed values come from the ONE grammar in `own-ir` that the strict door already delegated to, an analysis-level fact-parity family freezes every violation member with zero Python, the bridge maps BR-P3 in its BR-V1 place, and both protocol documents are promoted out of the exclusion ledger without regenerating either golden; **#259 final acceptance reached** — the last thing it owed was the coordinate-domain decision, and that landed Python-first: `spec/OwnIR.md` §4.2 bounds every `line` to `[0, 2147483647]` and every `column` to `[1, 2147483647]` (int32 is the line type of every consumer this project feeds; `0` stays legal as the reference's own absent sentinel), every line-bearing field is validated including the two §4.2 recorded as checked nowhere, the tolerant door degrades an out-of-domain coordinate rather than clamping it, the Rust door and bridge mirror all of it, and the four `verdict_boundary_*` controls are promoted out of the exclusion ledger — which now names only the two #294 OD-1 door controls, a declared boundary rather than open work. Not shadow mode, which is #260's acceptance. Every count is generated: `docs/generated/p022-cp1-census.md`, `docs/generated/p022-cp4-census.md`, `docs/generated/p022-coord-census.md`, `docs/generated/p022-cp5-inventory.md`, `docs/generated/p022-cp4b-mutations.md` and `docs/generated/p022-coord-mutations.md`); step 7a shadow-mode INFRASTRUCTURE complete (checkpoints 1–4: `ownlang/repro.py` + `own-shadow` — canonical same-input `OwnIR` identity, the reproduction-artifact format, the engine protocol, the `AnalysisTrace` (#269) with stable-ID normalization, first-divergence reduction), and #260's **acceptance decisions landed over the committed corpus**: the verdict layer is in reduction scope (the scope IS the layer order), acceptance is a field of its own beside the observation kind under a frozen `(layer, kind, class)` boundary policy the refusing engine declares structurally, canonical SARIF is compared as a DERIVED surface rather than a layer, artifact v3 attests the raw input and each engine's `consumed` (so the byte-level same-input invariant is proved rather than approximated by canonical identity), and a dev-only `own-shadow-engine` adapter plus a compare driver run the two engines over one byte sequence in CI. Compare mode over the committed corpus reports zero acceptance-unexplained at all three layers and on the derived SARIF, with the two #294 OD-1 typed-door boundaries declared by policy. **NOT #260's acceptance and not shadow mode**: its test matrix also names a five-repository sweep and large-solution controls, and neither is taken; a crash is never a fallback, and Python stays the public engine until #262's cutover. Every count is generated (`docs/generated/p022-shadow-census.md`, `docs/generated/p022-shadow-mutations.md`), the decisions are recorded verbatim in [the owner-decision ledger](../notes/p022-shadow-infra-owner-decisions.md), and the record is [the acceptance note](../notes/p022-shadow-acceptance.md) | +| [P-022](P-022-rust-core-migration.md) | Rust core migration: crate DAG, patterns, prior art, differential oracle (Python = golden) | in execution — steps 0–4 built (#214/#249); step 5a done (full diagnostic contract, #255 via #319/#320/#321); step 5b SARIF done (#256; `.ownreport.json` struck — a buffer report needing the AST, not a diagnostics surface); step 6a done (`spec/Bridge.md`, #258); step 6b complete at final acceptance (`own-lowered`/`own-bridge`, #259: lowering and MOS parity landed; strict-door validation complete with no known divergence — the first 0/0/0 proved to be the ledger agreeing with its own author, and the second omitted two families that a Python-first defensive-limit change (#326) had to close before the third could measure them; analysis wiring complete at the checkpoint-4 surface — `check_facts` through the real analyses, Layer 3 goldens built, with an executable exclusion ledger naming each declared boundary; **cp5 complete at its surface** — the replay compares EVERY `Finding` member (the BR-V4 wording matrix and the BR-V5 evidence slices included) and every refusal in full, and a second fixture family freezes the BR-V9 rendered surfaces byte for byte, all against goldens none of which was regenerated; **row 4b complete** — the obligation-protocol analysis (OBL001–005) is ported into `own-analysis`, its typed values come from the ONE grammar in `own-ir` that the strict door already delegated to, an analysis-level fact-parity family freezes every violation member with zero Python, the bridge maps BR-P3 in its BR-V1 place, and both protocol documents are promoted out of the exclusion ledger without regenerating either golden; **#259 final acceptance reached** — the last thing it owed was the coordinate-domain decision, and that landed Python-first: `spec/OwnIR.md` §4.2 bounds every `line` to `[0, 2147483647]` and every `column` to `[1, 2147483647]` (int32 is the line type of every consumer this project feeds; `0` stays legal as the reference's own absent sentinel), every line-bearing field is validated including the two §4.2 recorded as checked nowhere, the tolerant door degrades an out-of-domain coordinate rather than clamping it, the Rust door and bridge mirror all of it, and the four `verdict_boundary_*` controls are promoted out of the exclusion ledger — which now names only the two #294 OD-1 door controls, a declared boundary rather than open work. Not shadow mode, which is #260's acceptance. Every count is generated: `docs/generated/p022-cp1-census.md`, `docs/generated/p022-cp4-census.md`, `docs/generated/p022-coord-census.md`, `docs/generated/p022-cp5-inventory.md`, `docs/generated/p022-cp4b-mutations.md` and `docs/generated/p022-coord-mutations.md`); step 7a shadow-mode INFRASTRUCTURE complete (checkpoints 1–4: `ownlang/repro.py` + `own-shadow` — canonical same-input `OwnIR` identity, the reproduction-artifact format, the engine protocol, the `AnalysisTrace` (#269) with stable-ID normalization, first-divergence reduction), and #260's **acceptance decisions landed over the committed corpus**: the verdict layer is in reduction scope (the scope IS the layer order), acceptance is a field of its own beside the observation kind under a frozen `(layer, kind, class)` boundary policy the refusing engine declares structurally, canonical SARIF is compared as a DERIVED surface rather than a layer, artifact v3 attests the raw input and each engine's `consumed` (so the byte-level same-input invariant is proved rather than approximated by canonical identity), and a dev-only `own-shadow-engine` adapter plus a compare driver run the two engines over one byte sequence in CI. **#260's final acceptance is REACHED**: compare mode reports zero acceptance-unexplained over its full test matrix — the committed corpus, the C# samples, the `examples/` tree, the five pinned OSS repositories of #243 at their verified pins and the large-solution controls — at all three layers and on the derived SARIF, on byte-attested same input, with the two #294 OD-1 typed-door boundaries declared by policy. The sweep is ten documents over six targets, each extracted exactly once through `own-check.sh --emit-facts` and compared from those bytes; a repository is not covered because its extraction succeeded, so a run that compared zero documents fails, a declared target nothing reached fails, and the denominators are recorded per target. Taking the measurement found six harness defects and no engine divergence. Still **not** shadow mode achieved, **not** "P-022 done" and **not** "Rust is the default" — that is #262's cutover behind #261; a crash is never a fallback, Python stays the public engine, and no production behaviour changed. Every count is generated (`docs/generated/p022-shadow-sweep.md`, `docs/generated/p022-shadow-census.md`, `docs/generated/p022-shadow-mutations.md`), the decisions are recorded verbatim in [the owner-decision ledger](../notes/p022-shadow-infra-owner-decisions.md), and the records are [the sweep note](../notes/p022-shadow-sweep.md) and [the acceptance note](../notes/p022-shadow-acceptance.md) | | [P-023](P-023-architecture-guard.md) | Architecture guard (`Own.Arch`): rules.yaml intent model + dependency-graph gate + baseline ratchet | draft | | [P-024](P-024-security-audit-profile.md) | Security audit profile (external tools + SARIF adapters; rejects own scanner engine) | draft | | [P-025](P-025-obligation-protocols.md) | Obligation protocols (`Own.Protocols`): barrier-sensitive project invariants (OBL001–005) | first slice built (core + bridge + fixtures; extractor pending) | diff --git a/pyproject.toml b/pyproject.toml index 9b586724..8a9ed88d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ files = [ "tests/verdict_render_census.py", "tests/validation_census.py", "tests/verdict_surface_inventory.py", + "tests/shadow_sweep.py", "tests/test_obligation_fact_parity.py", "scripts/mutate_campaign.py", "scripts/shadow_compare.py", diff --git a/scripts/mutate_campaign.py b/scripts/mutate_campaign.py index bf61ae3f..4395a2ee 100644 --- a/scripts/mutate_campaign.py +++ b/scripts/mutate_campaign.py @@ -135,6 +135,11 @@ class Layer: PARSERS = ("cargo", "python-fail") +# Line endings, named rather than spelled inline: a source is matched against +# patterns written with LF and written back with the ending it arrived with. +LF = "\n" +CRLF = "\r\n" + @dataclass(frozen=True) class Definition: @@ -473,7 +478,8 @@ def workspace_packages(workspace: str) -> list[str]: """Every workspace member, from cargo itself — never a typed list.""" out = subprocess.run( ["cargo", "metadata", "--no-deps", "--format-version", "1"], - cwd=os.path.join(ROOT, workspace), check=True, capture_output=True, text=True, + cwd=os.path.join(ROOT, workspace), check=True, capture_output=True, + text=True, encoding="utf-8", errors="replace", ).stdout meta = json.loads(out) members = {str(m) for m in meta.get("workspace_members", [])} @@ -490,7 +496,15 @@ def parse_test_output(package: str, out: str) -> tuple[list[str], bool]: for line in out.splitlines(): m = _RUNNING.match(line) if m: - target = m.group(1) + # cargo prints the target with the host's separator, so on + # Windows it says `tests\repro.rs` where every campaign + # definition and every recorded result says `tests/repro.rs`. + # A catcher name is an IDENTITY, and one that depends on the + # platform that produced it makes `expected_catchers` silently + # unmatchable there — measured: five acc-1 mutations reported + # "expected catchers MISSED" while naming exactly the test + # that had been expected. + target = m.group(1).replace('\\', "/") continue m = _DOCTESTS.match(line) if m: @@ -520,8 +534,16 @@ def _run_layer(layer: Layer) -> tuple[list[str], bool, list[str]]: if layer.parser == "python-fail": # Never leave a .pyc behind: see the cache note in the module docstring. env["PYTHONDONTWRITEBYTECODE"] = "1" + # UTF-8 explicitly, and never the console codepage: `text=True` decodes with + # the locale encoding, so on a machine whose console is cp1251 a single + # non-ASCII byte anywhere in cargo's output raised UnicodeDecodeError and + # killed the campaign mid-run — measured, at mutation B03 of + # p022-shadow-acc-2. cargo and this repository's harnesses both emit UTF-8, + # and `errors="replace"` means a stray byte costs one character rather than + # the whole run. r = subprocess.run(list(layer.command), cwd=os.path.join(ROOT, layer.cwd), env=env, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding="utf-8", errors="replace") if layer.parser == "cargo": found, ce = parse_test_output(layer.id, r.stdout) else: @@ -565,15 +587,32 @@ def _layers_of(definition: Definition) -> tuple[Layer, ...]: for pkg in workspace_packages(workspace)) -def write_source(target: str, text: str) -> None: +def read_source(target: str) -> tuple[str, str]: + """A source's text with LF endings, and the ending the file actually uses. + + A campaign's patterns are written with LF, so the text they are matched + against is normalized; the ENDING is carried beside it so that writing the + file back — mutated or restored — reproduces the bytes that were there. + + Without that pair the harness round-tripped every target through the + platform's newline translation, and on a checkout whose working copy is + CRLF each campaign rewrote its targets and then correctly refused its own + result because the tree had changed. The tree HAD changed; what changed it + was the harness.""" + with open(os.path.join(ROOT, target), "rb") as f: + text = f.read().decode("utf-8") + return text.replace(CRLF, LF), (CRLF if CRLF in text else LF) + + +def write_source(target: str, text: str, ending: str = LF) -> None: """Write a mutated (or restored) source and drop any cached bytecode for it. CPython validates a `.pyc` by the source's integer mtime and size, so a same-size rewrite inside the same second leaves the stale bytecode valid and the interpreter runs the file that is no longer on disk.""" path = os.path.join(ROOT, target) - with open(path, "w", encoding="utf-8") as f: - f.write(text) + with open(path, "wb") as f: + f.write(text.replace(LF, ending).encode("utf-8")) if not target.endswith(".py"): return directory, name = os.path.split(path) @@ -792,13 +831,13 @@ def run_campaign(definition: Definition, allow_dirty: bool) -> Result: packages = [] if definition.layers else [x.id for x in layers] targets = sorted({m.target for m in definition.mutations}) pristine: dict[str, str] = {} + endings: dict[str, str] = {} for t in targets: - with open(os.path.join(ROOT, t), encoding="utf-8") as f: - pristine[t] = f.read() + pristine[t], endings[t] = read_source(t) def restore() -> None: for t, text in pristine.items(): - write_source(t, text) + write_source(t, text, endings[t]) print(f"{definition.control_id}: {definition.control_description}", flush=True) print(f" layers: {', '.join(x.id for x in layers)}", flush=True) @@ -830,7 +869,7 @@ def restore() -> None: outcomes.append(Outcome(m.id, "compile-error", (), 0.0, broken)) print(f" -> compile-error: {broken}", flush=True) continue - write_source(m.target, mutated) + write_source(m.target, mutated, endings[m.target]) t0 = time.monotonic() try: catchers, ce, unparsed = run_tests(definition) @@ -851,9 +890,8 @@ def restore() -> None: finally: restore() for t, text in pristine.items(): - with open(os.path.join(ROOT, t), encoding="utf-8") as f: - if f.read() != text: - raise CampaignError(f"{t} was not restored to its pristine content") + if read_source(t) != (text, endings[t]): + raise CampaignError(f"{t} was not restored to its pristine content") assert_tree_unchanged(baseline, "before recording the result") return Result( campaign=definition.campaign, diff --git a/scripts/render_checkpoint_status.py b/scripts/render_checkpoint_status.py index e0b4ec9b..d15bb22c 100644 --- a/scripts/render_checkpoint_status.py +++ b/scripts/render_checkpoint_status.py @@ -46,6 +46,12 @@ campaigns, through the same `summarize()` as cp4's. One interpreter for every campaign in the tree: two readings of one run is how two documents come to disagree about it. +* `docs/generated/p022-shadow-sweep.md` — #260's FINAL-ACCEPTANCE sweep: the + five pinned OSS repositories of #243, the large-solution controls and the + examples tree, from `tests/shadow_sweep.compute_sweep_summary()` over the + committed definition and one recorded run. The per-target denominators are + the point of the document: a repository is not covered because its + extraction succeeded. Determinism: nothing in a fragment depends on HEAD, the clock or the environment, so an unrelated commit never changes one. The campaign fragment @@ -89,6 +95,14 @@ summarize, ) from shadow_census import ShadowCensus, ShadowCensusError, compute_shadow_census # noqa: E402 +from shadow_sweep import DEFINITION as DEFINITION_PATH # noqa: E402 +from shadow_sweep import ( # noqa: E402 + TARGET_FIELDS, + SweepError, + SweepSummary, + compute_sweep_summary, + load_pair, +) from validation_census import ( # noqa: E402 ValidationCensus, ValidationCensusError, @@ -119,6 +133,7 @@ MUTATIONS_MD = "p022-cp4-mutations.md" SHADOW_CENSUS_MD = "p022-shadow-census.md" SHADOW_MUTATIONS_MD = "p022-shadow-mutations.md" +SHADOW_SWEEP_MD = "p022-shadow-sweep.md" CAMPAIGN = os.path.join(EVIDENCE, "p022-cp4-mutations.json") RESULT = os.path.join(EVIDENCE, "p022-cp4-mutations.result.json") # One campaign per shadow checkpoint: each stays frozen at what it measured, so @@ -132,6 +147,8 @@ "p022-shadow-acc-1"), ("acceptance 2 — the scope, the boundary policy, the derived surface and the " "driver (D-4..D-7, R-1, R-2)", "p022-shadow-acc-2"), + ("final acceptance — the driver's v2 surfaces and the sweep interpreter", + "p022-shadow-sweep-1"), ) # One campaign per cp5 sub-checkpoint, for the same reason the shadow slice has # one per checkpoint: a campaign stays frozen at what it measured, so a later @@ -625,6 +642,101 @@ def render_campaign_set(heading: str, blurb: str, campaigns: tuple[tuple[str, st return "\n".join(parts), problems + + +# --- step 7a: #260's final-acceptance sweep -------------------------------- + + +def render_sweep(summary: SweepSummary) -> str: + """The sweep, as the one document every count in the packet links to. + + The per-target block is not decoration: #250's fifth failure mode is a + green gate over an empty set, so the DENOMINATOR is rendered beside the + outcome for every target, and a target with nothing compared is visible + rather than absent.""" + document_rows = "\n".join( + f"| `{d.id}` | `{d.target}` | {d.extraction_mode} | `{d.target_commit[:12]}` | " + f"{d.raw_bytes} | `{d.raw_digest[:12]}` | `{d.canonical_digest[:12]}` | " + f"{d.reduction_outcome} | {d.derived_outcome} | {d.declared_boundary} | " + f"{d.unexplained} | {d.wall_clock_seconds:.2f} | {d.timeout_seconds:.0f} |" + for d in summary.documents) + target_rows = "\n".join("| `" + row[0] + "` | " + " | ".join(row[1:]) + " |" + for row in summary.targets) + totals_row = ("| **total** | " + + " | ".join(str(summary.totals[name]) for name in TARGET_FIELDS) + + " |") + adapter_rows = "\n".join(f"| `{digest}` | {size} |" + for digest, size in summary.adapters) + doc_head = " | ".join(( + "document", "target", "mode", "pin", "raw bytes", "raw sha256", + "canonical sha256", "reduction", "derived SARIF", "declared-boundary", + "unexplained", "wall clock (s)", "timeout (s)")) + doc_align = "|---|---|---|---|---:|---|---|---|---|---:|---:|---:|---:|" + target_head = " | ".join(( + "target", "extracted", "compared", "agreed", "diverged", + "execution failures", "input refusals", "input disagreements", + "declared-boundary", "unexplained")) + target_align = "|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|" + where = (f"[the workflow run]({summary.workflow_run_url})" + if summary.workflow_run_url + else "a local run (no workflow run URL)") + unexplained = summary.totals.get("acceptance_unexplained_observations", 0) + verdict = ( + "Every document agreed and no observation is acceptance-unexplained." + if unexplained == 0 and all(d.outcome == "agreed" for d in summary.documents) + else "**This run is NOT clean.** A document did not agree, or an " + "observation is acceptance-unexplained: that is a finding, it " + "stays on the record until it is resolved, and #260 stays open.") + return f"""{_header(_rel(DEFINITION_PATH) + " and its .result.json")} +# P-022 step 7a (#260) — the final-acceptance sweep + +The measurement the acceptance surfaces over the committed corpus deliberately +did not take: the five pinned OSS repositories of #243 at their pinned commits, +the large/multi-project solution controls, and the `examples/` tree. Every +document is one OwnIR byte sequence fed to BOTH engines and judged by compare +mode; the record is [`{_rel(DEFINITION_PATH)}`](../evidence/p022-shadow-sweep.json) +(what should be measured) and its `.result.json` (one actual run), read by +`tests/shadow_sweep.py`. {verdict} + +**What "covered" means here.** A repository is not covered because extraction +succeeded, and a solution is not covered because some project inside it emitted +OwnIR. Coverage is a recorded, non-empty set of documents fed byte-identically +to both engines and judged — per target, with the denominator on the record. +A run that compared zero documents is a failure, and so is a declared target +nothing reached. + +## The run + +| what | value | +|---|---| +| Own.NET commit | `{summary.source_commit}` | +| recorded at | {summary.recorded_at} | +| host | {summary.host} | +| where | {where} | +| driver | `shadow_compare_version` {summary.driver_version} | + +The adapter each leg executed, by digest — a path is not an identity, so a +stale build cannot stand in for the engine that was meant: + +| `own-shadow-engine` sha256 | bytes | +|---|---:| +{adapter_rows} + +## The documents + +| {doc_head} | +{doc_align} +{document_rows} + +## The denominators, per target + +| {target_head} | +{target_align} +{target_rows} +{totals_row} +""" + + def render_shadow_census(c: ShadowCensus) -> str: corpus_rows = "\n".join(f"| `tests/fixtures/{corpus}` | {n} |" for corpus, n in c.by_corpus) engine_rows = "\n".join(f"| `{eid}` | {produced} | {refused} | {full} | {partial} |" @@ -642,12 +754,13 @@ def render_shadow_census(c: ShadowCensus) -> str: return f"""{_header("tests/fixtures/repro/ (artifacts, traces, reductions)")} # P-022 step 7a — shadow-mode infrastructure: census -**Compare mode over the committed corpus — not #260's acceptance.** What is -measured here is every document this repository commits, at all three layers -and on the derived SARIF surface, on byte-attested same input. #260's test -matrix additionally names a five-repository sweep and large-solution controls; -neither is run here, and nothing below may be read as shadow mode having been -achieved, as P-022 being done, or as Rust being the default. +**Compare mode over the committed corpus — one leg of #260's test matrix.** +What is measured here is every document this repository commits, at all three +layers and on the derived SARIF surface, on byte-attested same input. The five +pinned OSS repositories, the large-solution controls and the examples tree are +the OTHER legs and have their own record +([`{SHADOW_SWEEP_MD}`]({SHADOW_SWEEP_MD})); nothing below may be read as shadow +mode having been achieved, as P-022 being done, or as Rust being the default. This document is the **live view** of the slice as it stands; the recorded mutation campaigns are their own fragment @@ -902,6 +1015,12 @@ def fragments() -> tuple[dict[str, str], list[str]]: shadow, shadow_problems = render_shadow_mutations() out[SHADOW_MUTATIONS_MD] = shadow problems.extend(f"mutation campaign {p}" for p in shadow_problems) + try: + sweep_summary = compute_sweep_summary(*load_pair()) + out[SHADOW_SWEEP_MD] = render_sweep(sweep_summary) + problems.extend(f"shadow sweep: {p}" for p in sweep_summary.problems) + except SweepError as e: + problems.extend(f"shadow sweep: {p}" for p in e.problems) cp5, cp5_problems = render_campaign_set( "# P-022 checkpoint 5 — mutation campaigns", "One campaign per sub-checkpoint, each frozen at what it measured. Every " @@ -983,7 +1102,8 @@ def main(argv: list[str]) -> int: if argv: print(f"checkpoint status fragments OK: {CENSUS_MD}, {CP1_CENSUS_MD}, " f"{COORD_CENSUS_MD}, {INVENTORY_MD}, {MUTATIONS_MD}, {CP5_MUTATIONS_MD}, " - f"{SHADOW_CENSUS_MD}, {SHADOW_MUTATIONS_MD} in sync with the evidence") + f"{SHADOW_CENSUS_MD}, {SHADOW_MUTATIONS_MD}, {SHADOW_SWEEP_MD} in " + f"sync with the evidence") return 0 diff --git a/scripts/shadow_compare.py b/scripts/shadow_compare.py index 60f26212..41b4b49d 100755 --- a/scripts/shadow_compare.py +++ b/scripts/shadow_compare.py @@ -58,24 +58,52 @@ R-2 — that information belongs in a failure report, not in a reproduction artifact, because there is no reproduction). +A `--manifest` run is the exception, and deliberately so: it *is* the record, +so every document's result is written, plus one run summary. An artifact is +still written only on mismatch. + +## Which adapter ran, said in a way a stale build cannot fake + +Every recorded comparison and every failure report names the adapter by +`sha256` and byte length as well as by path, computed from the file this run +actually executed. A path is not an identity: `rust/target/release/own-shadow- +engine` is a different program on Tuesday, and a run that reported agreement +against yesterday's build reported agreement about yesterday. The digest is +taken here rather than by the caller for the same reason `consumed` is taken by +each engine — an identity somebody else supplies is a claim, not a measurement. + +## The empty set is not agreement + +A run that compared **zero** documents FAILS, and so does a target whose +compare-attempted count is zero. #250's fifth failure mode is that a green gate +over an empty set is worse than a red one, because a red one at least says it +is awake; a repository is not covered because its extraction succeeded, and a +solution is not covered because some project inside it emitted OwnIR. The +denominators are therefore part of what a run reports, per target and in total. + Run: python scripts/shadow_compare.py --engine compare python scripts/shadow_compare.py --engine python python scripts/shadow_compare.py --engine rust python scripts/shadow_compare.py --engine compare --corpus (every committed document) + python scripts/shadow_compare.py --engine compare --manifest """ from __future__ import annotations import argparse +import hashlib import json import os import subprocess import sys +import time +from dataclasses import dataclass from typing import Any sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")) from ownlang.repro import ( + ACCEPTANCE_DECLARED, ACCEPTANCE_UNEXPLAINED, DERIVED_RENDERER_ONLY, ENGINE_PYTHON, @@ -98,14 +126,33 @@ # changed is not a capture this driver knows how to place in an artifact. ADAPTER_PROTOCOL = 1 +# The shape of the documents this driver writes. 1 → 2 adds the adapter's +# identity (`engine_binary_sha256` / `engine_binary_bytes`) to every result and +# every failure report, and the manifest run's summary. The ARTIFACT format +# (`REPRO_VERSION`) is a different number and is not touched. +SHADOW_COMPARE_VERSION = 2 + +# The manifest schema this driver speaks. A manifest is provenance as well as a +# work list: each entry says which target and which commit the document came +# from, by which extraction mode and command, and what its bytes hash to. +MANIFEST_SCHEMA = 1 +MANIFEST_FIELDS = ("source", "target", "target_commit", "extraction_mode", + "extraction_command", "facts_sha256") + # Where the port's adapter is looked for, in order, when `--engine-binary` is # not given. A fixed, short list that is REPORTED in the result rather than # guessed at: "which binary did this run actually use" is part of what a -# recorded comparison means. +# recorded comparison means — and since v2 it is reported by DIGEST, so a +# candidate that resolves to a stale build can no longer stand in unnoticed. +# The `.exe` spellings are here because without them the whole list names files +# that cannot exist on Windows, which is not a safer default, only a quieter +# one. ENGINE_BINARY_ENV = "OWN_SHADOW_ENGINE" ENGINE_BINARY_CANDIDATES = ( os.path.join(ROOT, "rust", "target", "release", "own-shadow-engine"), + os.path.join(ROOT, "rust", "target", "release", "own-shadow-engine.exe"), os.path.join(ROOT, "rust", "target", "debug", "own-shadow-engine"), + os.path.join(ROOT, "rust", "target", "debug", "own-shadow-engine.exe"), ) OUTCOME_AGREED = "agreed" @@ -160,6 +207,27 @@ def resolve_engine_binary(explicit: str | None) -> str: f"not a comparison.") +def engine_identity(binary: str) -> dict[str, Any]: + """The adapter, named by digest and length as well as by path. + + Taken from the file this run is about to execute, in this program, rather + than accepted from whoever invoked it: an identity somebody else supplies + is a claim. A path that cannot be read is a usage error and not a run — a + driver that could not say WHICH engine it ran should not be reporting what + that engine said.""" + try: + with open(binary, "rb") as f: + raw = f.read() + except OSError as e: + raise SystemExit( + f"shadow_compare: cannot read the adapter at {binary!r} to name it " + f"({e}). A recorded comparison names the engine by sha256, not only " + f"by path, so an adapter this driver cannot hash is one it must not " + f"run.") from e + return {"path": binary, "sha256": hashlib.sha256(raw).hexdigest(), + "bytes": len(raw)} + + def read_once(path: str) -> bytes: """The **only** read of the input in this program. @@ -170,7 +238,68 @@ def read_once(path: str) -> bytes: return f.read() -def run_port(raw: bytes, binary: str, timeout: float) -> dict[str, Any]: +def _source_label(path: str) -> str: + """The name a result records for an input. + + Repository-relative when the input is inside the repository, and the + absolute path otherwise. It used to be `os.path.relpath` unconditionally, + which RAISES on Windows for a path on another drive — the sweep's facts + documents live outside the checkout, so the driver crashed on the first one + before either engine ran. A label is not worth an exception.""" + try: + rel = os.path.relpath(path, ROOT) + except ValueError: + return os.path.abspath(path).replace(os.sep, "/") + if rel == os.pardir or rel.startswith(os.pardir + os.sep): + return os.path.abspath(path).replace(os.sep, "/") + return rel.replace(os.sep, "/") + + +def _spawn(binary: str, raw: bytes, timeout: float + ) -> tuple[bytes, bytes, int, bool]: + """Run the adapter over `raw` and come back within `timeout`, always. + + `subprocess.run(timeout=…)` is not enough, and the timeout control is what + said so: it kills the CHILD and then waits for the pipes to close, so any + grandchild still holding them keeps the driver blocked — a timeout that + never returns is not a timeout, and owner decision R-2 makes a timeout a + run-level hard failure the driver has to *report*. So the child gets its own + process group (POSIX) or is killed as a tree (Windows). + + The real adapter starts nothing, which is exactly why this had never been + hit: the property is about what the driver guarantees, not about what + today's adapter happens to do.""" + process = subprocess.Popen( + [binary], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + start_new_session=(os.name != "nt")) + try: + stdout, stderr = process.communicate(input=raw, timeout=timeout) + except subprocess.TimeoutExpired: + _kill_tree(process) + try: + stdout, stderr = process.communicate(timeout=30) + except subprocess.TimeoutExpired: # pragma: no cover - the pipes are gone + stdout, stderr = b"", b"" + return stdout, stderr, process.returncode or 0, True + return stdout, stderr, process.returncode, False + + +def _kill_tree(process: subprocess.Popen[bytes]) -> None: + """The adapter and everything it started, on both platforms.""" + if sys.platform == "win32": + subprocess.run(["taskkill", "/F", "/T", "/PID", str(process.pid)], + capture_output=True, check=False) + else: + import signal as _signal + try: + os.killpg(os.getpgid(process.pid), _signal.SIGKILL) + except OSError: + pass + process.kill() + + +def run_port(raw: bytes, adapter: dict[str, Any], timeout: float) -> dict[str, Any]: """Pipe the captured bytes to the port's adapter and read back its capture. Every way this can go wrong is a **run-level hard failure** and raises @@ -178,33 +307,37 @@ def run_port(raw: bytes, binary: str, timeout: float) -> dict[str, Any]: that is not JSON, or an envelope this driver does not speak. None of them is a layer refusal, and none of them may be replaced by the reference's result.""" + binary = str(adapter["path"]) + def failure(detail: str, **extra: Any) -> ExecutionFailure: return ExecutionFailure({ - "engine": ENGINE_RUST, "binary": binary, "detail": detail, + "engine": ENGINE_RUST, "binary": binary, + "engine_binary_sha256": adapter["sha256"], + "engine_binary_bytes": adapter["bytes"], + "detail": detail, "exit_code": None, "signal": None, "timed_out": False, "timeout_seconds": timeout, "stderr": "", **extra}) try: - completed = subprocess.run( - [binary], input=raw, capture_output=True, timeout=timeout, check=False) - except subprocess.TimeoutExpired as e: + stdout, stderr_raw, code, timed_out = _spawn(binary, raw, timeout) + except OSError as e: + raise failure(f"the port's adapter could not be run: {e}") from e + if timed_out: raise failure( f"the port's adapter did not finish within {timeout}s", timed_out=True, - stderr=_text(e.stderr)) from e - except OSError as e: - raise failure(f"the port's adapter could not be run: {e}") from e - stderr = _text(completed.stderr) - if completed.returncode != 0: + stderr=_text(stderr_raw)) + stderr = _text(stderr_raw) + if code != 0: # A negative return code is a signal on POSIX. Recorded as a signal # rather than folded into the exit code, because "killed by SIGSEGV" # and "exited 11" are different events and only one of them is a crash. - signal = -completed.returncode if completed.returncode < 0 else None + signal = -code if code < 0 else None raise failure( - f"the port's adapter exited {completed.returncode}", - exit_code=completed.returncode, signal=signal, stderr=stderr) + f"the port's adapter exited {code}", + exit_code=code, signal=signal, stderr=stderr) try: - envelope = json.loads(completed.stdout.decode("utf-8")) + envelope = json.loads(stdout.decode("utf-8")) except (UnicodeDecodeError, json.JSONDecodeError) as e: raise failure( f"the port's adapter wrote something that is not a JSON capture: {e}", @@ -243,7 +376,7 @@ def run_reference(raw: bytes) -> dict[str, Any]: return {"entry": entry, "canonical_error": None} -def compare(raw: bytes, source: str, binary: str, +def compare(raw: bytes, source: str, adapter: dict[str, Any], timeout: float) -> tuple[str, dict[str, Any]]: """One byte sequence through both engines, and the verdict about them. @@ -252,15 +385,18 @@ def compare(raw: bytes, source: str, binary: str, """ identity = hash_bytes(raw) result: dict[str, Any] = { - "shadow_compare_version": 1, + "shadow_compare_version": SHADOW_COMPARE_VERSION, "source": source, "input": {"raw": identity}, - "engine_binary": binary, + "engine_binary": adapter["path"], + "engine_binary_sha256": adapter["sha256"], + "engine_binary_bytes": adapter["bytes"], + "timeout_seconds": timeout, } # The PORT first, then the reference. Adversarial on purpose: if anything # in this driver re-read the file, an input that changed between the two # runs would show up as a `consumed` mismatch instead of passing unnoticed. - port = run_port(raw, binary, timeout) + port = run_port(raw, adapter, timeout) envelope = port["envelope"] reference = run_reference(raw) @@ -317,6 +453,13 @@ def compare(raw: bytes, source: str, binary: str, result["artifact"] = artifact return OUTCOME_DIVERGED, result + # Both engines derived this identity from the same bytes, independently, + # and the check above is what proved they agree about it. Recording it here + # rather than only inside the artifact matters because the artifact is + # written on MISMATCH only: without this line a green run would name the + # bytes it compared and not the document they are. + result["input"]["canonical"] = artifact["input"]["canonical"] + traces = project_traces(artifact, source) reduction = reduce_traces(traces) derived = derived_outcome(artifact, reduction) @@ -429,6 +572,181 @@ def _domain_refusal_names() -> set[str]: if isinstance(e.get("name"), str)} +# --- the manifest run ----------------------------------------------------- + + +@dataclass(frozen=True) +class Document: + """One document of a manifest run: what to compare, and where it came from. + + `source` is both the file to read and the label the result records. One + field rather than two on purpose — a record whose label can drift from the + file it names is a record that can attribute a result to the wrong + document. `id` is the document's name in the sweep's definition and in the + aggregation; it defaults to the file's basename, which is enough for a + hand run and not enough for CI, where every leg writes `facts.json`.""" + + id: str + source: str + path: str + target: str + target_commit: str + extraction_mode: str + extraction_command: str + facts_sha256: str + timeout_seconds: float + + +def load_manifest(path: str, default_timeout: float + ) -> tuple[list[Document], list[str]]: + """The work list, with its provenance, and the targets it claims to cover. + + Every document field is required. A relative `source` resolves against the + MANIFEST's directory, not the process's, so a manifest and the documents it + names travel together. + + The optional top-level `targets` is what makes "a target this run did not + reach" *representable*: without it every target in the summary is one some + document already named, so a skipped target would silently not appear + rather than appear at zero. When it is given it is the authority — a + document naming a target it does not list is a manifest error.""" + try: + with open(path, encoding="utf-8") as f: + data = json.load(f) + except (OSError, json.JSONDecodeError) as e: + raise SystemExit(f"shadow_compare: cannot read the manifest {path!r}: {e}") from e + if not isinstance(data, dict): + raise SystemExit(f"shadow_compare: {path}: a manifest is a JSON object") + if data.get("schema") != MANIFEST_SCHEMA: + raise SystemExit( + f"shadow_compare: {path}: manifest schema {data.get('schema')!r}, this " + f"driver speaks {MANIFEST_SCHEMA} — regenerate it rather than reading " + f"around the difference") + entries = data.get("documents") + if not isinstance(entries, list): + raise SystemExit(f"shadow_compare: {path}: 'documents' must be a list") + declared = data.get("targets", []) + if not isinstance(declared, list) or not all( + isinstance(t, str) and t for t in declared): + raise SystemExit( + f"shadow_compare: {path}: 'targets' must be a list of non-empty " + f"strings when it is given") + base = os.path.dirname(os.path.abspath(path)) + out: list[Document] = [] + for i, entry in enumerate(entries): + where = f"{path}: documents[{i}]" + if not isinstance(entry, dict): + raise SystemExit(f"shadow_compare: {where}: must be an object") + missing = [name for name in MANIFEST_FIELDS + if not isinstance(entry.get(name), str) or not entry[name]] + if missing: + raise SystemExit( + f"shadow_compare: {where}: missing or empty {', '.join(missing)}. " + f"A manifest entry is the document's provenance; a run that cannot " + f"say which target and commit a document came from is not a record.") + timeout = entry.get("timeout_seconds", default_timeout) + if not isinstance(timeout, (int, float)) or isinstance(timeout, bool) or timeout <= 0: + raise SystemExit( + f"shadow_compare: {where}: 'timeout_seconds' must be a positive " + f"number (it is recorded per document, never inherited silently)") + source = str(entry["source"]) + doc_id = entry.get("id") or os.path.basename(source) + if not isinstance(doc_id, str) or not doc_id: + raise SystemExit( + f"shadow_compare: {where}: 'id' must be a non-empty string") + out.append(Document( + id=doc_id, + source=source, + path=source if os.path.isabs(source) else os.path.join(base, source), + target=str(entry["target"]), + target_commit=str(entry["target_commit"]), + extraction_mode=str(entry["extraction_mode"]), + extraction_command=str(entry["extraction_command"]), + facts_sha256=str(entry["facts_sha256"]), + timeout_seconds=float(timeout))) + seen: set[str] = set() + for doc in out: + if doc.id in seen: + raise SystemExit( + f"shadow_compare: {path}: two documents share the id {doc.id!r}. " + f"The id is how a document is joined to the sweep definition and " + f"to its own result, so two of them is one document lost.") + seen.add(doc.id) + if declared: + stray = sorted({d.target for d in out} - set(declared)) + if stray: + raise SystemExit( + f"shadow_compare: {path}: documents name target(s) " + f"{', '.join(stray)} that 'targets' does not declare — the " + f"declared list is the denominator, so it may not be a subset " + f"of what happened to be measured") + return out, [str(t) for t in declared] + + +def read_and_verify(documents: list[Document]) -> tuple[list[bytes], list[str]]: + """Every document read ONCE, hashed, and checked — before any engine runs. + + Reading first and hashing the buffer is deliberate, and it is stronger than + hashing the file and then reading it: there is no window between the check + and the use in which the file could change, and the bytes the engines + receive are the very bytes that were verified. It also keeps the one-read + invariant intact, which a separate hashing pass would not. + + Every problem is collected rather than raised at the first: a manifest with + two stale digests should say so once.""" + buffers: list[bytes] = [] + problems: list[str] = [] + for doc in documents: + try: + raw = read_once(doc.path) + except OSError as e: + buffers.append(b"") + problems.append(f"{doc.source}: cannot be read ({e})") + continue + buffers.append(raw) + digest = hashlib.sha256(raw).hexdigest() + if digest != doc.facts_sha256: + problems.append( + f"{doc.source}: the manifest says the extracted document hashes to " + f"{doc.facts_sha256}, the file hashes to {digest} — this is not " + f"the document that was extracted, and no engine may see it") + return buffers, problems + + +def _zeroed_target(target: str, extracted: int) -> dict[str, Any]: + return {"target": target, "documents_extracted": extracted, + "compare_attempted": 0, "agreed": 0, "diverged": 0, + "execution_failures": 0, "input_refusals": 0, + "input_disagreements": 0, + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0} + + +_TALLY_FIELD = { + OUTCOME_AGREED: "agreed", + OUTCOME_DIVERGED: "diverged", + OUTCOME_EXECUTION_FAILURE: "execution_failures", + OUTCOME_INPUT_REFUSED: "input_refusals", + OUTCOME_INPUT_DISAGREEMENT: "input_disagreements", +} +_SUMMED = ("documents_extracted", "compare_attempted", "agreed", "diverged", + "execution_failures", "input_refusals", "input_disagreements", + "declared_boundary_observations", + "acceptance_unexplained_observations") + + +def _own_net_commit() -> str | None: + """The commit this driver ran from, for the record. `None` outside a + checkout — a missing provenance line is better than an invented one.""" + try: + out = subprocess.run(["git", "-C", ROOT, "rev-parse", "HEAD"], + capture_output=True, text=True, check=False, timeout=30) + except (OSError, subprocess.SubprocessError): + return None + sha = out.stdout.strip() + return sha if out.returncode == 0 and sha else None + + def main(argv: list[str]) -> int: parser = argparse.ArgumentParser( prog="shadow_compare", @@ -438,21 +756,39 @@ def main(argv: list[str]) -> int: default="compare") parser.add_argument("--corpus", action="store_true", help="compare over every committed facts document") + parser.add_argument("--manifest", default=None, + help="a JSON manifest of documents with their provenance " + "(target, commit, extraction mode and command, " + "facts_sha256, and the targets the run claims to " + "cover); writes one result per document plus a run " + "summary") parser.add_argument("--engine-binary", default=None, help="the own-shadow-engine adapter to run") parser.add_argument("--timeout", type=float, default=DEFAULT_TIMEOUT_SECONDS, - help="hard timeout per adapter run, in seconds") + help="hard timeout per adapter run, in seconds; a " + "manifest entry may name its own") parser.add_argument("--out", default=None, help="directory for artifacts, reductions and failure " - "reports (written on divergence or failure only)") + "reports (written on divergence or failure only; a " + "--manifest run also writes every result and the " + "run summary)") parser.add_argument("--quiet", action="store_true", help="print one line per case instead of the full result") args = parser.parse_args(argv) - if args.corpus == bool(args.path): - print("shadow_compare: give exactly one of or --corpus", + chosen = [name for name, given in (("", bool(args.path)), + ("--corpus", args.corpus), + ("--manifest", args.manifest is not None)) + if given] + if len(chosen) != 1: + print("shadow_compare: give exactly one of , --corpus or " + "--manifest" + (f" (got {', '.join(chosen)})" if chosen else ""), file=sys.stderr) return EXIT_USAGE + if args.manifest is not None and args.engine != "compare": + print("shadow_compare: --manifest is a compare-mode run; --engine " + "python and --engine rust take one path", file=sys.stderr) + return EXIT_USAGE if args.engine == "python": if args.path is None: @@ -466,14 +802,14 @@ def main(argv: list[str]) -> int: sys.stdout.write(_render(reference["entry"])) return 0 - binary = resolve_engine_binary(args.engine_binary) + adapter = engine_identity(resolve_engine_binary(args.engine_binary)) if args.engine == "rust": if args.path is None: print("shadow_compare: --engine rust takes a path", file=sys.stderr) return EXIT_USAGE try: - port = run_port(read_once(args.path), binary, args.timeout) + port = run_port(read_once(args.path), adapter, args.timeout) except ExecutionFailure as e: print(f"shadow_compare: {e.report['detail']}", file=sys.stderr) if e.report.get("stderr"): @@ -482,59 +818,79 @@ def main(argv: list[str]) -> int: sys.stdout.write(_render(port["envelope"]["engine"])) return 0 + if args.manifest is not None: + return _run_manifest(args, adapter) + return _run_paths(args, adapter) + + +def _one(raw: bytes, source: str, adapter: dict[str, Any], timeout: float, + refusals: set[str], out_dir: str | None, + ) -> tuple[str, dict[str, Any], float]: + """One document, end to end: compare, classify, report, time it. + + Shared by both run shapes so that the corpus gate and the sweep cannot + develop two readings of one outcome.""" + stem = os.path.basename(source) + stem = stem[: -len(".facts.json")] if stem.endswith(".facts.json") else stem + started = time.monotonic() + try: + outcome, result = compare(raw, source, adapter, timeout) + except ExecutionFailure as e: + elapsed = time.monotonic() - started + report = dict(e.report) + report["shadow_compare_version"] = SHADOW_COMPARE_VERSION + report["outcome"] = OUTCOME_EXECUTION_FAILURE + report["source"] = source + report["wall_clock_seconds"] = round(elapsed, 3) + # The raw input rides in the FAILURE REPORT, never in a reproduction + # artifact (owner decision R-2): there is no reproduction, because + # no capture was produced. + report["input"] = {"raw": encode_raw(raw)} + written = _write(out_dir, f"{_safe_name(source)}.failure.json", report) + print(f"FAIL[execution] {source}: {report['detail']}" + + (f" (report: {written})" if written else ""), file=sys.stderr) + if report.get("stderr"): + print(report["stderr"], file=sys.stderr, end="") + return OUTCOME_EXECUTION_FAILURE, report, elapsed + elapsed = time.monotonic() - started + + # A document the ledger declares unnameable by both engines is expected + # to be refused; anything else there is the finding. + if stem in refusals: + if outcome == OUTCOME_INPUT_REFUSED: + outcome = OUTCOME_AGREED + result["outcome"] = OUTCOME_AGREED + result["detail"] = ("a declared domain refusal: both engines " + "refuse to name it, which is the control") + else: + result["detail"] = ( + f"{result.get('detail')} — and this document is a DECLARED " + f"domain refusal, so anything but a refusal is the finding") + outcome = OUTCOME_DIVERGED + result["outcome"] = OUTCOME_DIVERGED + result["wall_clock_seconds"] = round(elapsed, 3) + if outcome != OUTCOME_AGREED: + written = _write(out_dir, f"{_safe_name(source)}.compare.json", result) + print(f"FAIL[{outcome}] {source}: {result.get('detail')}" + + (f" (report: {written})" if written else ""), file=sys.stderr) + return outcome, result, elapsed + + +def _run_paths(args: argparse.Namespace, adapter: dict[str, Any]) -> int: + """`--corpus`, or one positional path.""" refusals = _domain_refusal_names() paths = _corpus() if args.corpus else [args.path] worst = 0 tally: dict[str, int] = {} for path in paths: - source = os.path.relpath(path, ROOT).replace(os.sep, "/") - stem = os.path.basename(path)[: -len(".facts.json")] - raw = read_once(path) - try: - outcome, result = compare(raw, source, binary, args.timeout) - except ExecutionFailure as e: - report = dict(e.report) - report["shadow_compare_version"] = 1 - report["outcome"] = OUTCOME_EXECUTION_FAILURE - report["source"] = source - # The raw input rides in the FAILURE REPORT, never in a reproduction - # artifact (owner decision R-2): there is no reproduction, because - # no capture was produced. - report["input"] = {"raw": encode_raw(raw)} - written = _write(args.out, f"{_safe_name(source)}.failure.json", report) - print(f"FAIL[execution] {source}: {report['detail']}" - + (f" (report: {written})" if written else ""), file=sys.stderr) - if report.get("stderr"): - print(report["stderr"], file=sys.stderr, end="") - tally[OUTCOME_EXECUTION_FAILURE] = tally.get( - OUTCOME_EXECUTION_FAILURE, 0) + 1 - worst = max(worst, EXIT[OUTCOME_EXECUTION_FAILURE]) - continue - - # A document the ledger declares unnameable by both engines is expected - # to be refused; anything else there is the finding. - if stem in refusals: - if outcome == OUTCOME_INPUT_REFUSED: - outcome = OUTCOME_AGREED - result["outcome"] = OUTCOME_AGREED - result["detail"] = ("a declared domain refusal: both engines " - "refuse to name it, which is the control") - else: - result["detail"] = ( - f"{result.get('detail')} — and this document is a DECLARED " - f"domain refusal, so anything but a refusal is the finding") - outcome = OUTCOME_DIVERGED - result["outcome"] = OUTCOME_DIVERGED - + source = _source_label(path) + outcome, result, _ = _one(read_once(path), source, adapter, args.timeout, + refusals, args.out) tally[outcome] = tally.get(outcome, 0) + 1 worst = max(worst, EXIT[outcome]) - if outcome != OUTCOME_AGREED: - written = _write(args.out, f"{_safe_name(source)}.compare.json", result) - print(f"FAIL[{outcome}] {source}: {result.get('detail')}" - + (f" (report: {written})" if written else ""), file=sys.stderr) - elif not args.quiet and not args.corpus: + if outcome == OUTCOME_AGREED and not args.quiet and not args.corpus: sys.stdout.write(_render(result)) - elif args.corpus and not args.quiet: + elif outcome == OUTCOME_AGREED and args.corpus and not args.quiet: reduction = result.get("reduction") if reduction is None: print(f"ok {source}: {result['detail']}") @@ -544,6 +900,123 @@ def main(argv: list[str]) -> int: if args.corpus or args.quiet: print(f"shadow compare over {len(paths)} document(s): " + ", ".join(f"{n} {name}" for name, n in sorted(tally.items()))) + if not paths: + print("shadow_compare: this run compared ZERO documents, which is a " + "failure and not agreement — a gate over an empty set is worse " + "than a red one", file=sys.stderr) + return EXIT_USAGE + return worst + + +def _run_manifest(args: argparse.Namespace, adapter: dict[str, Any]) -> int: + """A multi-document run with provenance, and the denominators to prove it. + + Every document is read and verified before any engine runs; then each is + compared, each result is written, and one summary carries the per-target + and total counts, the adapter's identity, this driver's version and the + Own.NET commit.""" + documents, declared = load_manifest(args.manifest, args.timeout) + buffers, problems = read_and_verify(documents) + if not documents: + problems.append("the manifest lists no documents, so this run compared " + "ZERO of them — a failure, never agreement") + if problems: + for p in problems: + print(f"FAIL[manifest] {p}", file=sys.stderr) + print("shadow_compare: no engine was run — the manifest is the record, " + "and a record that does not describe the bytes on disk cannot be " + "repaired by comparing them anyway", file=sys.stderr) + return EXIT_USAGE + + refusals = _domain_refusal_names() + per_target: dict[str, dict[str, Any]] = { + target: _zeroed_target(target, 0) for target in declared} + for doc in documents: + row = per_target.setdefault(doc.target, _zeroed_target(doc.target, 0)) + row["documents_extracted"] += 1 + + worst = 0 + records: list[dict[str, Any]] = [] + for doc, raw in zip(documents, buffers, strict=True): + outcome, result, elapsed = _one(raw, doc.source, adapter, + doc.timeout_seconds, refusals, args.out) + row = per_target[doc.target] + row["compare_attempted"] += 1 + row[_TALLY_FIELD[outcome]] += 1 + acceptance = (result.get("reduction") or {}).get( + "classification", {}).get("by_acceptance", {}) + row["declared_boundary_observations"] += int( + acceptance.get(ACCEPTANCE_DECLARED, 0)) + row["acceptance_unexplained_observations"] += int( + acceptance.get(ACCEPTANCE_UNEXPLAINED, 0)) + worst = max(worst, EXIT[outcome]) + result["document_id"] = doc.id + _write(args.out, f"{_safe_name(doc.id)}.result.json", result) + records.append({ + "id": doc.id, + "source": doc.source, + "target": doc.target, + "target_commit": doc.target_commit, + "extraction_mode": doc.extraction_mode, + "extraction_command": doc.extraction_command, + "facts_sha256": doc.facts_sha256, + "timeout_seconds": doc.timeout_seconds, + "outcome": outcome, + "wall_clock_seconds": round(elapsed, 3), + }) + if not args.quiet: + print(f"{'ok' if outcome == OUTCOME_AGREED else outcome} " + f"{doc.source} [{doc.target}/{doc.extraction_mode}]") + + # A target the run never reached is a FAILED target, not a passed + # repository: extraction succeeding says nothing about a comparison. + empty = [t for t, row in sorted(per_target.items()) + if row["compare_attempted"] == 0] + totals = {name: sum(int(row[name]) for row in per_target.values()) + for name in _SUMMED} + summary: dict[str, Any] = { + "shadow_compare_version": SHADOW_COMPARE_VERSION, + "manifest": args.manifest, + "declared_targets": declared, + "engine_binary": adapter["path"], + "engine_binary_sha256": adapter["sha256"], + "engine_binary_bytes": adapter["bytes"], + "own_net_commit": _own_net_commit(), + "documents": records, + "targets": [per_target[t] for t in sorted(per_target)], + "totals": totals, + "targets_with_no_comparison": empty, + } + if empty: + worst = max(worst, EXIT_USAGE) + for t in empty: + print(f"FAIL[manifest] target {t!r} had ZERO documents compared — a " + f"target is not covered because its extraction ran", + file=sys.stderr) + # The backstop for the same rule, and it is UNREACHABLE while the check + # above stands: an empty manifest returns before this, and a manifest with + # documents cannot reach zero attempts. It is kept because it states the + # rule at the place the number actually exists, and it is reachable exactly + # when the first check is removed — which is what a mutation does. That is + # why the control pins the FIRST check's own wording rather than the exit + # code: two enforcement points of one rule and a control that cannot tell + # them apart is a control that proves nothing (the shape §5.1 of the + # acceptance note records). + if totals["compare_attempted"] == 0: + worst = max(worst, EXIT_USAGE) + print("shadow_compare: the totals say ZERO documents were compared, " + "which is a failure and not agreement", file=sys.stderr) + summary["outcome"] = (OUTCOME_AGREED if worst == 0 else "not-agreed") + _write(args.out, "summary.json", summary) + print(f"shadow compare over {totals['compare_attempted']} document(s) in " + f"{len(per_target)} target(s): {totals['agreed']} agreed, " + f"{totals['diverged']} diverged, {totals['execution_failures']} " + f"execution-failure, {totals['input_refusals']} input-refused, " + f"{totals['input_disagreements']} input-disagreement; " + f"{totals['declared_boundary_observations']} declared-boundary and " + f"{totals['acceptance_unexplained_observations']} " + f"acceptance-unexplained observation(s); adapter " + f"{adapter['sha256'][:12]}") return worst diff --git a/spec/Bridge.md b/spec/Bridge.md index 5278b053..021152eb 100644 --- a/spec/Bridge.md +++ b/spec/Bridge.md @@ -439,7 +439,7 @@ identifiers and each engine's per-layer output; the `AnalysisTrace` (#269) that normalizes those outputs into a walkable shape; and a reducer that names the first place two engines part company. -Seven points belong to this spec rather than to those notes. +Eight points belong to this spec rather than to those notes. 1. Every layer in an artifact is projected through the **tolerant** door on one in-memory document — mixing doors across layers would mean the three entries @@ -491,12 +491,30 @@ Seven points belong to this spec rather than to those notes. divergence`, `not-comparable` — are distinct because "the renderers disagree" and "there was nothing to compare" are different findings. -Nothing there is shadow mode. What compare mode measures is the **committed -corpus**: zero acceptance-unexplained at all three layers and on the derived -SARIF, on byte-attested same input, with the two OD-1 typed-door boundaries -declared by policy. #260's acceptance additionally requires the five-repository -sweep and the large-solution controls, neither of which is taken — -see [the acceptance note](../docs/notes/p022-shadow-acceptance.md). +Nothing there is shadow mode. What compare mode measures is a **named set of +documents**, and #260's final acceptance is the point at which that set became +its whole test matrix: the committed corpus, the C# samples, the `examples/` +tree, the five pinned OSS repositories of #243 at their verified pins, and the +largest solution of every one of them that has one — zero acceptance-unexplained +at all three layers and on the derived SARIF, on byte-attested same input, with +the two OD-1 typed-door boundaries declared by policy. An eighth point belongs +to this spec because it is what makes that sentence checkable rather than +believed: + +8. **A comparison is only evidence over a set somebody wrote down first.** A + repository is not covered because its extraction succeeded, and a solution is + not covered because some project inside it emitted OwnIR. A sweep therefore + carries a **definition** — the targets with their pins, the documents with + their extraction mode and verbatim command — and one **recorded run**; the + run is checked against the definition, so a run that is short, long, or + measured at another commit is refused. A run that compared zero documents + fails, and so does a declared target it never reached. The denominators are + part of what a run reports, per target and in total. Every engine a run + executed is named by `sha256` and byte length, not by path. + +See [the sweep note](../docs/notes/p022-shadow-sweep.md) for what was measured +and [the acceptance note](../docs/notes/p022-shadow-acceptance.md) for the +committed-corpus half beneath it. Regeneration: each layer gets a `--write` mode mirroring `tests/test_cfg_fixtures.py`; a stale committed fixture is a red build; the diff --git a/tests/shadow_sweep.py b/tests/shadow_sweep.py new file mode 100644 index 00000000..ba4c8a3d --- /dev/null +++ b/tests/shadow_sweep.py @@ -0,0 +1,485 @@ +#!/usr/bin/env python3 +"""The one interpretation of #260's sweep: its definition, one recorded run. + +The counterpart to `tests/shadow_census.py` for the measurement the acceptance +surfaces over the committed corpus deliberately did not take — the five pinned +OSS repositories of #243, the large-solution controls and the `examples/` tree. +The status renderer and the test gate both read this module, because two +readings of one run is how two documents come to disagree about it. + +## What a definition is, and what a result is + +The **definition** (`docs/evidence/p022-shadow-sweep.json`) is what SHOULD be +measured: the targets with their pins, the documents with their extraction mode +and verbatim command, the timeout each carries, the driver version it expects +and how the port's adapter is built. It is the DENOMINATOR, and that is the +whole reason it exists as a separate file: a run that reports only what it +happened to measure cannot be short. + +The **result** (`….result.json`) is one actual run. A re-run replaces it whole; +it is never patched, because a document patched into a run that did not produce +it is a claim wearing a measurement's clothes. + +## What this refuses + +Nothing here decides whether two engines agree — the driver does that, and this +reads what it wrote. What this decides is whether the RUN is evidence: + +* an empty run, or a target whose compare-attempted count is zero — #250's + fifth failure mode, and the reason the denominators are recorded per target; +* a document the definition declares and the result does not carry; +* a document the result carries that the definition does not declare; +* a document measured at a commit, in a mode, or by a command other than the + one the definition names — the run then measured something, but not this; +* a driver version, or an adapter identity, the result cannot name; +* a `source_commit` that is not an ancestor of HEAD; +* any document outcome other than `agreed`, and any acceptance-unexplained + observation, including one hiding inside a document recorded as agreed. + +## `--collect` + +CI runs one leg per document and each leg writes its own run summary. Collect +merges them into the single result document above, so that the record a +workflow produces and the record a local run produces are the same shape, +assembled by the same code. + +Run: python tests/shadow_sweep.py (the committed pair) + python tests/shadow_sweep.py --result (the definition vs another run) + python tests/shadow_sweep.py --collect --write +""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from dataclasses import dataclass, field +from typing import Any + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +EVIDENCE = os.path.join(ROOT, "docs", "evidence") +DEFINITION = os.path.join(EVIDENCE, "p022-shadow-sweep.json") +RESULT = os.path.join(EVIDENCE, "p022-shadow-sweep.result.json") + +SCHEMA = 1 +# The only document outcome this sweep may report. `declared-boundary` is an +# ACCEPTANCE of an observation inside a document (owner decision D-5), never a +# document outcome — a document whose observations are all agreed-or-declared +# is reported by the driver as `agreed`, which is the value checked here. +OUTCOME_AGREED = "agreed" +SHA256 = re.compile(r"^[0-9a-f]{64}$") + +# The counters a target row carries. Named here rather than derived from +# whatever a result happens to contain: a denominator that only exists when +# somebody wrote it is not a denominator. +TARGET_FIELDS = ("documents_extracted", "compare_attempted", "agreed", + "diverged", "execution_failures", "input_refusals", + "input_disagreements", "declared_boundary_observations", + "acceptance_unexplained_observations") + + +class SweepError(Exception): + def __init__(self, problems: list[str]) -> None: + super().__init__("; ".join(problems)) + self.problems = problems + + +@dataclass +class DocumentRow: + id: str + target: str + target_commit: str + extraction_mode: str + outcome: str + raw_digest: str + raw_bytes: int + canonical_digest: str + reduction_outcome: str + derived_outcome: str + declared_boundary: int + unexplained: int + timeout_seconds: float + wall_clock_seconds: float + + +@dataclass +class SweepSummary: + """Every figure the sweep's status surfaces may show, and nothing else.""" + + sweep: str = "" + source_commit: str = "" + recorded_at: str = "" + host: str = "" + workflow_run_url: str | None = None + driver_version: int = 0 + adapters: tuple[tuple[str, int], ...] = () + documents: tuple[DocumentRow, ...] = () + targets: tuple[tuple[str, ...], ...] = () + totals: dict[str, int] = field(default_factory=dict) + problems: list[str] = field(default_factory=list) + + +def _load(path: str) -> Any: + with open(path, encoding="utf-8") as f: + return json.load(f) + + +def _git_ok(*args: str) -> bool: + try: + done = subprocess.run(["git", "-C", ROOT, *args], capture_output=True, + text=True, check=False, timeout=60) + except (OSError, subprocess.SubprocessError): + return False + return done.returncode == 0 + + +def provenance_problems(source_commit: str) -> list[str]: + """The gate's view of a run's provenance, and deliberately the same shape a + mutation campaign's has: the recorded commit must exist in this tree and be + an ancestor of HEAD. It refuses a run that describes a history this tree + does not contain, without making the record depend on HEAD's content.""" + if not source_commit: + return ["the run records no source commit, so nothing says which tree " + "it measured"] + if not _git_ok("rev-parse", "--git-dir"): + return [f"cannot verify the provenance of {source_commit[:12]}: not a " + f"git checkout"] + if not _git_ok("cat-file", "-e", f"{source_commit}^{{commit}}"): + return [f"source commit {source_commit[:12]} does not exist in this " + f"repository — a rebased or deleted branch? re-run the sweep"] + if not _git_ok("merge-base", "--is-ancestor", source_commit, "HEAD"): + return [f"source commit {source_commit[:12]} is not an ancestor of HEAD " + f"— the run describes a history this tree does not contain; " + f"re-run the sweep"] + return [] + + +def compute_sweep_summary(definition: dict[str, Any], + result: dict[str, Any]) -> SweepSummary: + """Definition + one run → the summary, with every problem it has.""" + problems: list[str] = [] + for name, doc, expected in (("definition", definition, SCHEMA), + ("result", result, SCHEMA)): + if doc.get("schema") != expected: + problems.append(f"the {name} declares schema " + f"{doc.get('schema')!r}, this interpreter reads " + f"{expected}") + declared = {str(d["id"]): d for d in definition.get("documents", [])} + measured = {str(d["id"]): d for d in result.get("documents", [])} + + if not measured: + problems.append( + "the run carries NO documents: it compared zero of them, which is a " + "failure and never agreement (a green gate over an empty set is " + "worse than a red one)") + for doc_id in sorted(set(declared) - set(measured)): + problems.append( + f"{doc_id}: declared by the definition and absent from the run — a " + f"document nobody measured is not a document that agreed") + for doc_id in sorted(set(measured) - set(declared)): + problems.append( + f"{doc_id}: measured by the run and not declared by the definition; " + f"the definition is the denominator and may not be a subset of what " + f"happened to be measured") + + want_driver = definition.get("driver_version") + if result.get("driver_version") != want_driver: + problems.append( + f"the run names driver version {result.get('driver_version')!r}, the " + f"definition expects {want_driver!r}") + + adapters: list[tuple[str, int]] = [] + for entry in result.get("adapters", []): + digest = str(entry.get("sha256", "")) + size = entry.get("bytes") + if not SHA256.match(digest) or not isinstance(size, int) or size <= 0: + problems.append( + f"an adapter is recorded as {entry!r}: a recorded comparison " + f"names the engine by a sha256 and a byte length, because a " + f"path is not an identity") + continue + adapters.append((digest, size)) + if not adapters: + problems.append( + "the run names no adapter at all — nothing says which engine it " + "compared the reference against") + + rows: list[DocumentRow] = [] + for doc_id in sorted(measured): + m = measured[doc_id] + d = declared.get(doc_id) + if d is not None: + for field_name in ("target", "extraction_mode", "extraction_command"): + if m.get(field_name) != d.get(field_name): + problems.append( + f"{doc_id}: the run recorded {field_name}=" + f"{m.get(field_name)!r}, the definition declares " + f"{d.get(field_name)!r} — the run measured something, " + f"but not this") + # A null pin in the definition means "this repository, at the commit + # the run was taken on" — the `examples/` document, whose target IS + # the tree being measured. It is a check rather than an exemption: + # the recorded pin must then be the run's own source commit. + want_pin = d.get("target_commit") + if want_pin is None: + want_pin = result.get("source_commit") + if m.get("target_commit") != want_pin: + problems.append( + f"{doc_id}: the run recorded target_commit=" + f"{m.get('target_commit')!r}, expected {want_pin!r} — the " + f"run measured something, but not this") + outcome = str(m.get("outcome", "")) + unexplained = int(m.get("acceptance_unexplained_observations", 0) or 0) + if outcome != OUTCOME_AGREED: + problems.append( + f"{doc_id}: outcome {outcome!r}. Only 'agreed' is an outcome " + f"this sweep may report; a divergence is a finding and stays on " + f"the record until it is resolved") + if unexplained: + problems.append( + f"{doc_id}: {unexplained} acceptance-unexplained observation(s) " + f"in a document recorded as {outcome!r}") + raw = m.get("raw") or {} + canonical = m.get("canonical") or {} + if not SHA256.match(str(raw.get("digest", ""))): + problems.append(f"{doc_id}: no raw byte identity is recorded, so " + f"nothing says which bytes were compared") + rows.append(DocumentRow( + id=doc_id, + target=str(m.get("target", "")), + target_commit=str(m.get("target_commit", "")), + extraction_mode=str(m.get("extraction_mode", "")), + outcome=outcome, + raw_digest=str(raw.get("digest", "")), + raw_bytes=int(raw.get("bytes", 0) or 0), + canonical_digest=str(canonical.get("digest", "")), + reduction_outcome=str(m.get("reduction_outcome", "")), + derived_outcome=str(m.get("derived_outcome", "")), + declared_boundary=int( + m.get("declared_boundary_observations", 0) or 0), + unexplained=unexplained, + timeout_seconds=float(m.get("timeout_seconds", 0) or 0), + wall_clock_seconds=float(m.get("wall_clock_seconds", 0) or 0))) + + target_rows = {str(t["target"]): t for t in result.get("targets", [])} + for target in sorted({str(t["target"]) for t in definition.get("targets", [])}): + row = target_rows.get(target) + if row is None: + problems.append( + f"target {target!r} is declared by the definition and missing " + f"from the run — a skipped target is not a passed repository") + continue + if int(row.get("compare_attempted", 0) or 0) == 0: + problems.append( + f"target {target!r} had ZERO documents compared: a target is " + f"not covered because its extraction ran") + extracted = int(row.get("documents_extracted", 0) or 0) + attempted = int(row.get("compare_attempted", 0) or 0) + if extracted != attempted: + problems.append( + f"target {target!r}: {extracted} document(s) extracted and " + f"{attempted} compared — the gap is a document that was skipped") + for target in sorted(set(target_rows) - { + str(t["target"]) for t in definition.get("targets", [])}): + problems.append(f"target {target!r} appears in the run and not in the " + f"definition") + + totals = {name: int((result.get("totals") or {}).get(name, 0) or 0) + for name in TARGET_FIELDS} + recomputed = {name: sum(int(r.get(name, 0) or 0) for r in target_rows.values()) + for name in TARGET_FIELDS} + for name in TARGET_FIELDS: + if totals[name] != recomputed[name]: + problems.append( + f"totals[{name!r}] is {totals[name]}, the per-target rows sum to " + f"{recomputed[name]} — a total that is not the sum of its parts " + f"is a number somebody typed") + problems.extend(provenance_problems(str(result.get("source_commit", "")))) + + return SweepSummary( + sweep=str(definition.get("sweep", "")), + source_commit=str(result.get("source_commit", "")), + recorded_at=str(result.get("recorded_at", "")), + host=str(result.get("host", "")), + workflow_run_url=(str(result["workflow_run_url"]) + if result.get("workflow_run_url") else None), + driver_version=int(result.get("driver_version", 0) or 0), + adapters=tuple(adapters), + documents=tuple(rows), + targets=tuple( + tuple([target] + [str(target_rows[target].get(name, 0)) + for name in TARGET_FIELDS]) + for target in sorted(target_rows)), + totals=totals, + problems=problems) + + +def load_pair(definition_path: str = DEFINITION, + result_path: str = RESULT) -> tuple[dict[str, Any], dict[str, Any]]: + for path in (definition_path, result_path): + if not os.path.exists(path): + raise SweepError([f"{os.path.relpath(path, ROOT)}: missing — the " + f"sweep is definition AND one recorded run"]) + return _load(definition_path), _load(result_path) + + +# --- assembling one record out of the legs of a run ------------------------ + + +def collect(directory: str, workflow_run_url: str | None, + definition_path: str = DEFINITION) -> dict[str, Any]: + """Merge every leg's run summary and per-document result into one record. + + CI runs one leg per document; a local run does all of them at once. Both + write the same files, so both are collected by this one function — the + record a workflow produces and the record a hand run produces cannot drift + into different shapes.""" + import hashlib + import platform + import time + + summaries: list[dict[str, Any]] = [] + details: dict[str, dict[str, Any]] = {} + for dirpath, _dirnames, filenames in os.walk(directory): + for name in sorted(filenames): + path = os.path.join(dirpath, name) + if name == "summary.json": + summaries.append(_load(path)) + elif name.endswith(".result.json"): + doc = _load(path) + key = str(doc.get("document_id") or "") + if key: + details[key] = doc + if not summaries: + raise SweepError([f"{directory}: no run summary found. A sweep record is " + f"assembled from what the driver wrote, never typed."]) + + documents: list[dict[str, Any]] = [] + targets: dict[str, dict[str, Any]] = {} + adapters: dict[str, int] = {} + versions: set[int] = set() + commits: set[str] = set() + for summary in summaries: + digest = str(summary.get("engine_binary_sha256", "")) + if digest: + adapters[digest] = int(summary.get("engine_binary_bytes", 0) or 0) + versions.add(int(summary.get("shadow_compare_version", 0) or 0)) + if summary.get("own_net_commit"): + commits.add(str(summary["own_net_commit"])) + for row in summary.get("targets", []): + into = targets.setdefault( + str(row["target"]), + {"target": str(row["target"]), **dict.fromkeys(TARGET_FIELDS, 0)}) + for name in TARGET_FIELDS: + into[name] = int(into[name]) + int(row.get(name, 0) or 0) + for record in summary.get("documents", []): + doc_id = str(record.get("id", "")) + detail = details.get(doc_id, {}) + reduction = detail.get("reduction") or {} + acceptance = (reduction.get("classification") or {}).get( + "by_acceptance", {}) + documents.append({ + "id": doc_id, + "source": record.get("source"), + "target": record.get("target"), + "target_commit": record.get("target_commit"), + "extraction_mode": record.get("extraction_mode"), + "extraction_command": record.get("extraction_command"), + "facts_sha256": record.get("facts_sha256"), + "raw": (detail.get("input") or {}).get("raw"), + "canonical": (detail.get("input") or {}).get("canonical"), + "outcome": record.get("outcome"), + "timeout_seconds": record.get("timeout_seconds"), + "reduction_outcome": reduction.get("outcome"), + "by_kind": (reduction.get("classification") or {}).get("by_kind"), + "declared_boundary_observations": int( + acceptance.get("declared-boundary", 0) or 0), + "acceptance_unexplained_observations": int( + acceptance.get("unexplained", 0) or 0), + "derived_outcome": (detail.get("derived") or {}).get("outcome"), + "wall_clock_seconds": record.get("wall_clock_seconds"), + }) + with open(definition_path, "rb") as f: + definition_sha = hashlib.sha256(f.read()).hexdigest() + return { + "schema": SCHEMA, + "comment": ("One recorded run of the #260 sweep, assembled by " + "tests/shadow_sweep.py --collect from what " + "scripts/shadow_compare.py wrote. Raw facts only: " + "identities, outcomes, denominators, provenance. Counts are " + "derived by scripts/render_checkpoint_status.py; regenerate " + "this file by re-running the sweep, never by hand."), + "sweep": "p022-shadow-sweep", + "definition": os.path.relpath(definition_path, ROOT).replace(os.sep, "/"), + "definition_sha256": definition_sha, + "source_commit": sorted(commits)[0] if len(commits) == 1 else "", + "recorded_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "host": f"{platform.system()} {platform.machine()}", + "workflow_run_url": workflow_run_url, + "driver_version": sorted(versions)[0] if len(versions) == 1 else 0, + "adapters": [{"sha256": d, "bytes": n} for d, n in sorted(adapters.items())], + "documents": sorted(documents, key=lambda d: str(d["id"])), + "targets": [targets[t] for t in sorted(targets)], + "totals": {name: sum(int(t[name]) for t in targets.values()) + for name in TARGET_FIELDS}, + } + + +def main(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="shadow_sweep", + description="Interpret #260's sweep definition and one recorded run.") + parser.add_argument("--result", default=RESULT, + help="the recorded run to read (default: the committed one)") + parser.add_argument("--definition", default=DEFINITION) + parser.add_argument("--collect", default=None, + help="assemble one record from a run directory instead") + parser.add_argument("--write", default=None, + help="where --collect writes the assembled record") + parser.add_argument("--workflow-run-url", default=None) + args = parser.parse_args(argv) + + if args.collect is not None: + try: + record = collect(args.collect, args.workflow_run_url, args.definition) + except SweepError as e: + # A failure, and a legible one: the aggregation job reaches this + # when every leg was skipped, which is exactly the shape #250's + # fifth failure mode takes in CI. + for p in e.problems: + print(f"FAIL[shadow-sweep]: {p}") + return 1 + text = json.dumps(record, indent=2, ensure_ascii=False) + "\n" + if args.write: + with open(args.write, "w", encoding="utf-8", newline="\n") as f: + f.write(text) + print(f"wrote {args.write}: {len(record['documents'])} document(s), " + f"{len(record['targets'])} target(s)") + else: + sys.stdout.write(text) + return 0 + + try: + definition, result = load_pair(args.definition, args.result) + except SweepError as e: + for p in e.problems: + print(f"FAIL[shadow-sweep]: {p}") + return 1 + summary = compute_sweep_summary(definition, result) + for p in summary.problems: + print(f"FAIL[shadow-sweep]: {p}") + if summary.problems: + return 1 + print(f"shadow sweep OK: {len(summary.documents)} document(s) over " + f"{len(summary.targets)} target(s), all agreed; " + f"{summary.totals['acceptance_unexplained_observations']} " + f"acceptance-unexplained observation(s); recorded at " + f"{summary.source_commit[:12]}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/test_checkpoint_status.py b/tests/test_checkpoint_status.py index 1962ba08..b29d7b9e 100644 --- a/tests/test_checkpoint_status.py +++ b/tests/test_checkpoint_status.py @@ -7,7 +7,8 @@ `scripts/render_checkpoint_status.py` (the verdict ledger census through `tests/verdict_census.py`; the cp5 surface inventory through `tests/verdict_surface_inventory.py`; the step-7a -census through `tests/shadow_census.py`; every recorded mutation campaign +census through `tests/shadow_census.py`; #260's final-acceptance sweep through +`tests/shadow_sweep.py`; every recorded mutation campaign through `scripts/mutate_campaign.py`). This module runs its `--check` in-process, so a change to the evidence without regenerating the fragments — or a campaign result that no longer matches its definition, was taken on a dirty @@ -56,6 +57,7 @@ SHADOW_CAMPAIGNS, SHADOW_CENSUS_MD, SHADOW_MUTATIONS_MD, + SHADOW_SWEEP_MD, check, ) @@ -97,7 +99,8 @@ def run() -> int: print(f"checkpoint status fragments OK: {CENSUS_MD}, {CP1_CENSUS_MD}, " f"{COORD_CENSUS_MD}, {INVENTORY_MD}, " f"{MUTATIONS_MD}, {CP4B_MUTATIONS_MD}, {CP5_MUTATIONS_MD}, {COORD_MUTATIONS_MD}, " - f"{SHADOW_CENSUS_MD}, {SHADOW_MUTATIONS_MD} in sync with the evidence; " + f"{SHADOW_CENSUS_MD}, {SHADOW_MUTATIONS_MD}, {SHADOW_SWEEP_MD} in sync " + f"with the evidence; " f"{len(DEFINITIONS)} campaign definitions still anchor") return 0 diff --git a/tests/test_mutate_campaign.py b/tests/test_mutate_campaign.py new file mode 100644 index 00000000..748bb737 --- /dev/null +++ b/tests/test_mutate_campaign.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""The mutation harness's own controls: bytes, and a catcher's identity. + +One rule, and it earned a test the hard way. `scripts/mutate_campaign.py` reads +each mutation target, rewrites it, and puts it back; it then refuses to record a +result if the tree changed. Reading and writing through Python's text mode makes +those two steps disagree on any checkout whose working copy uses CRLF: every +campaign rewrote its targets' line endings and then correctly refused its own +run. The tree HAD changed — what changed it was the harness — so no campaign +could be recorded on that platform at all, which is exactly the kind of silence +this discipline exists to break. + +The control drives `read_source` / `write_source` directly, with both endings, +because the property is about bytes on disk and nothing above them can observe +it: the harness's own "was it restored?" check compares TEXT, and passed +throughout. + +Run: python tests/test_mutate_campaign.py + python tests/run_tests.py (auto-discovered with every other test_*.py) +""" + +from __future__ import annotations + +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + "..", "scripts")) + +from mutate_campaign import ( + CRLF, + LF, + Layer, + _run_layer, + parse_test_output, + read_source, + write_source, +) + + +def run() -> int: + fails: list[str] = [] + with tempfile.TemporaryDirectory() as work: + for name, ending in (("lf", LF), ("crlf", CRLF)): + path = os.path.join(work, f"target_{name}.py") + body = ("A = 1" + ending + "B = 2" + ending).encode("utf-8") + with open(path, "wb") as f: + f.write(body) + + text, seen = read_source(path) + if seen != ending: + fails.append(f"{name}: read_source saw {seen!r}, the file uses " + f"{ending!r}") + if text != "A = 1\nB = 2\n": + fails.append(f"{name}: the text a pattern is matched against is " + f"{text!r}; a campaign's patterns are written with " + f"LF, so it must be normalized") + + # RESTORE: byte-for-byte, or the harness is what made the tree dirty. + write_source(path, text, seen) + with open(path, "rb") as f: + restored = f.read() + if restored != body: + fails.append(f"{name}: restoring rewrote the file " + f"({restored!r} != {body!r}) — a campaign that does " + f"this refuses its own result, and the tree it " + f"blames is one it changed itself") + + # A MUTATION keeps the file's own ending too: a mutated target that + # differs from the original in more than the mutation is a mutation + # nobody can attribute. + write_source(path, text.replace("A = 1", "A = 9"), seen) + with open(path, "rb") as f: + mutated = f.read() + want = ("A = 9" + ending + "B = 2" + ending).encode("utf-8") + if mutated != want: + fails.append(f"{name}: a mutated write produced {mutated!r}, " + f"expected {want!r}") + + # The default is LF, so a caller that forgets the ending cannot silently + # convert a file rather than leave it alone. + path = os.path.join(work, "default.py") + write_source(path, "A = 1\n") + with open(path, "rb") as f: + if f.read() != b"A = 1\n": + fails.append("write_source's default ending is not LF") + + # A CATCHER NAME IS AN IDENTITY, and it may not depend on the platform + # that produced it. cargo prints its target with the host separator, so on + # Windows the same failing test was recorded as `own-shadow/tests\\repro.rs + # ::…` while every definition and every committed result says + # `own-shadow/tests/repro.rs::…`. Five acc-1 mutations reported "expected + # catchers MISSED" while naming exactly the test that had been expected. + windows = ( + " Running tests\\repro.rs (target\\debug\\deps\\repro-1.exe)\n" + "test verify_refuses_each_structural_violation ... FAILED\n") + posix = ( + " Running tests/repro.rs (target/debug/deps/repro-1)\n" + "test verify_refuses_each_structural_violation ... FAILED\n") + want = ["own-shadow/tests/repro.rs::verify_refuses_each_structural_violation"] + for label, out in (("windows", windows), ("posix", posix)): + found, _ = parse_test_output("own-shadow", out) + if found != want: + fails.append(f"{label}: cargo output names catcher(s) {found}, " + f"expected {want} — a catcher whose spelling " + f"depends on the host makes expected_catchers " + f"unmatchable there") + + + # A LAYER'S OUTPUT IS UTF-8, whatever the console codepage says. `text=True` + # alone decodes with the locale encoding, so on a cp1251 console one + # non-ASCII byte in cargo's output raised UnicodeDecodeError and took the + # whole campaign with it — measured, mid-run, at a mutation of + # p022-shadow-acc-2. The child here prints an em dash for exactly that + # reason: it is the character the repository's own test names contain. + layer = Layer(id="own-shadow", cwd=".", parser="cargo", command=( + sys.executable, "-c", + "import sys; sys.stdout.reconfigure(encoding='utf-8'); " + "print(' Running tests/repro.rs (target/debug/deps/repro-1)'); " + "print('a name with an em dash \u2014 here'); " + "print('test verify_refuses_each_structural_violation ... FAILED'); " + "sys.exit(101)")) + try: + found, _compile_error, _unparsed = _run_layer(layer) + except UnicodeDecodeError as e: + fails.append(f"a layer whose output is not ASCII killed the harness: {e}") + else: + want = ["own-shadow/tests/repro.rs::verify_refuses_each_structural_violation"] + if found != want: + fails.append(f"a non-ASCII layer named catcher(s) {found}, expected " + f"{want}") + + if fails: + for f in fails: + print(f"FAIL[campaign-harness]: {f}") + return 1 + print("campaign harness OK: 12 controls held (a source is read with its own " + "ending and matched as LF; restoring and mutating both reproduce that " + "ending byte-for-byte; the default is LF; a cargo catcher has one " + "identity on both hosts; a layer whose output is not ASCII is read " + "rather than fatal)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(run()) diff --git a/tests/test_shadow_compare.py b/tests/test_shadow_compare.py index 32c7be1e..1742de60 100644 --- a/tests/test_shadow_compare.py +++ b/tests/test_shadow_compare.py @@ -61,6 +61,7 @@ EXIT_EXECUTION_FAILURE = 2 EXIT_INPUT_REFUSED = 3 EXIT_INPUT_DISAGREEMENT = 4 +EXIT_USAGE = 5 REQUIRED_ENV = "OWN_SHADOW_COMPARE_REQUIRED" @@ -87,6 +88,33 @@ def _adapter() -> str | None: return None +def _double_engine(directory: str) -> str: + """The stand-in, in a form this platform can actually start. + + The driver runs its adapter as ONE argv entry, because the real adapter is + a binary that takes no arguments (owner decision R-1). POSIX starts + `fake_shadow_engine.py` from its shebang. Windows cannot start a `.py` at + all — `CreateProcess` does not consult file associations — so every + double-driven control failed there with `WinError 193`, and the group whose + whole point is that it "runs everywhere, including the Python-only test + matrix" did not run on Windows at all. A one-line launcher beside it is the + smallest thing that keeps the driver's contract intact and the controls + running on both platforms; it is also the file whose digest the driver then + records, which is correct — it is the file that ran.""" + if os.name != "nt": + return FAKE + launcher = os.path.join(directory, "fake_shadow_engine.cmd") + with open(launcher, "w", encoding="ascii", newline="\r\n") as f: + f.write("@echo off\n") + f.write(f'"{sys.executable}" "{FAKE}" %*\n') + return launcher + + +def _digest(path: str) -> str: + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + def _run(args: list[str], *, engine: str, mode: str | None = None, extra_env: dict[str, str] | None = None, timeout: float = 300.0) -> subprocess.CompletedProcess[str]: @@ -106,6 +134,8 @@ def _double_controls() -> list[tuple[str, str]]: """The driver's reaction to a child that misbehaves (owner decision R-2).""" fails: list[tuple[str, str]] = [] with tempfile.TemporaryDirectory() as out: + fake_engine = _double_engine(out) + def expect(label: str, mode: str, code: int, needle: str, artifact_forbidden: bool = True, extra_env: dict[str, str] | None = None, @@ -114,7 +144,7 @@ def expect(label: str, mode: str, code: int, needle: str, case_out = os.path.join(out, mode) try: done = _run([*(args or [BASE]), "--out", case_out], - engine=FAKE, mode=mode, extra_env=extra_env, + engine=fake_engine, mode=mode, extra_env=extra_env, timeout=timeout) except subprocess.TimeoutExpired: fails.append(("compare-control", @@ -206,7 +236,7 @@ def expect(label: str, mode: str, code: int, needle: str, target = os.path.join(out, "rewritten.facts.json") os.makedirs(out, exist_ok=True) shutil.copyfile(BASE, target) - done = _run([target], engine=FAKE, mode="rewrite_input", + done = _run([target], engine=fake_engine, mode="rewrite_input", extra_env={"OWN_FAKE_ENGINE_REWRITE": target}) if done.returncode != EXIT_AGREED: fails.append(("compare-one-read", @@ -255,6 +285,228 @@ def expect(label: str, mode: str, code: int, needle: str, return fails +def _manifest_controls() -> list[tuple[str, str]]: + """The v2 surfaces: the adapter's identity, a manifest run's provenance and + denominators, and the rule that an empty set is not agreement. + + Double-driven throughout — the subject is the driver's bookkeeping, not + what the two engines say about a document, so every entry names the same + committed document under a different source label. Three copies of one file + is exactly the point: the counts must come from the OUTCOMES, and nothing + else about these three differs.""" + fails: list[tuple[str, str]] = [] + with open(BASE, "rb") as f: + base_raw = f.read() + base_digest = hashlib.sha256(base_raw).hexdigest() + + with tempfile.TemporaryDirectory() as work: + engine = _double_engine(work) + engine_digest = _digest(engine) + # A SECOND adapter on disk, one line different, so "the digest of the + # file that ran" and "the digest of some adapter" cannot be the same + # answer. Without it, a driver that hashed the wrong file would still + # produce a digest and every assertion below would pass. + decoy = os.path.join(work, "decoy_engine" + os.path.splitext(engine)[1]) + with open(engine, "rb") as f: + decoy_bytes = f.read() + b"\n@rem a different adapter\n" + with open(decoy, "wb") as f: + f.write(decoy_bytes) + decoy_digest = hashlib.sha256(decoy_bytes).hexdigest() + if decoy_digest == engine_digest: + fails.append(("compare-adapter-identity", + "the decoy adapter hashes to the same value as the one " + "that runs, so this control proves nothing")) + + def manifest(name: str, documents: list[dict[str, object]], + targets: list[str] | None = None) -> str: + path = os.path.join(work, f"{name}.manifest.json") + body: dict[str, object] = {"schema": 1, "documents": documents} + if targets is not None: + body["targets"] = targets + with open(path, "w", encoding="utf-8") as f: + json.dump(body, f, indent=2) + return path + + def entry(source: str, target: str, + digest: str = base_digest) -> dict[str, object]: + path = os.path.join(work, source) + with open(path, "wb") as f: + f.write(base_raw) + return {"source": path, "target": target, + "target_commit": "0" * 40, + "extraction_mode": "directory-walk", + "extraction_command": "scripts/own-check.sh --emit-facts …", + "facts_sha256": digest, "timeout_seconds": 120.0} + + # 1. THE adapter identity, in a green result: the digest is of the file + # that ran, and provably not of the other adapter beside it. + done = _run([BASE], engine=engine, mode="faithful") + result = _result_of(done, "the adapter identity", fails) + if result is not None: + if result.get("engine_binary_sha256") != engine_digest: + fails.append(("compare-adapter-identity", + f"the result names adapter digest " + f"{result.get('engine_binary_sha256')}, the file the " + f"driver was told to run hashes to {engine_digest} — " + f"a recorded comparison that names the wrong engine " + f"is a recorded comparison of nothing")) + if result.get("engine_binary_sha256") == decoy_digest: + fails.append(("compare-adapter-identity", + "the result names the DECOY adapter's digest: the " + "identity was taken from a file other than the one " + "that ran")) + if result.get("engine_binary_bytes") != os.path.getsize(engine): + fails.append(("compare-adapter-identity", + f"the result names " + f"{result.get('engine_binary_bytes')} adapter bytes, " + f"the file that ran is {os.path.getsize(engine)}")) + if result.get("shadow_compare_version") != 2: + fails.append(("compare-adapter-identity", + f"the result declares shadow_compare_version " + f"{result.get('shadow_compare_version')!r}; the " + f"adapter identity is what version 2 IS")) + + # 2. ...and in a FAILURE REPORT, where it matters at least as much: a + # crash you cannot attribute to a build is a crash you cannot chase. + crash_out = os.path.join(work, "crash") + done = _run([BASE, "--out", crash_out], engine=engine, mode="crash") + reports = ([n for n in sorted(os.listdir(crash_out)) + if n.endswith(".failure.json")] + if os.path.isdir(crash_out) else []) + if done.returncode != EXIT_EXECUTION_FAILURE or not reports: + fails.append(("compare-adapter-identity", + f"a crashing adapter produced no failure report (exit " + f"{done.returncode})")) + else: + report = json.loads(_read(crash_out, reports[0])) + if report.get("engine_binary_sha256") != engine_digest: + fails.append(("compare-adapter-identity", + f"the failure report names adapter digest " + f"{report.get('engine_binary_sha256')}, expected " + f"{engine_digest}")) + + # 3. An EMPTY manifest is a failure, not agreement — #250's fifth + # failure mode, as an executable rule. + empty_out = os.path.join(work, "empty") + done = _run(["--manifest", manifest("empty", []), "--out", empty_out], + engine=engine, mode="faithful") + if done.returncode != EXIT_USAGE: + fails.append(("compare-empty-set", + f"an empty manifest exited {done.returncode}, expected " + f"{EXIT_USAGE}: a run that compared zero documents " + f"reported something other than failure")) + # The wording of the check that is SUPPOSED to fire, not merely "some + # check fired": the totals backstop says ZERO too, so a needle both + # messages match would let a mutation of either one survive. + if "the manifest lists no documents" not in done.stdout + done.stderr: + fails.append(("compare-empty-set", + f"an empty run did not refuse it at the manifest, " + f"where the rule is stated. Output: " + f"{(done.stdout + done.stderr)[:400]}")) + + # 4. A DIGEST MISMATCH stops the run before any engine is started. The + # double is in `crash` mode: had one been started, the exit would be + # the execution-failure code rather than the usage one, and a failure + # report would exist. + stale_out = os.path.join(work, "stale") + stale = manifest("stale", [entry("stale.facts.json", "T", "0" * 64)]) + done = _run(["--manifest", stale, "--out", stale_out], + engine=engine, mode="crash") + blob = done.stdout + done.stderr + if done.returncode != EXIT_USAGE: + fails.append(("compare-manifest-digest", + f"a manifest whose facts_sha256 does not match the file " + f"exited {done.returncode}, expected {EXIT_USAGE}")) + if "no engine was run" not in blob: + fails.append(("compare-manifest-digest", + f"the driver did not say that it refused before running " + f"an engine. Output: {blob[:400]}")) + written = sorted(os.listdir(stale_out)) if os.path.isdir(stale_out) else [] + if written: + fails.append(("compare-manifest-digest", + f"the driver wrote {written} for a manifest it refused: " + f"the digest check happens BEFORE any engine runs, so " + f"there is nothing to report about")) + + # 5. The counts come from the OUTCOMES. The same three documents twice: + # once with a faithful double (all agree) and once with one that + # attests nothing (all diverge). A summary that counted entries + # rather than outcomes would be identical in both runs. + documents = [entry("a.facts.json", "alpha"), + entry("b.facts.json", "alpha"), + entry("c.facts.json", "beta")] + good = manifest("good", documents, targets=["alpha", "beta"]) + for mode, code, agreed, diverged in (("faithful", EXIT_AGREED, 3, 0), + ("no_consumed", EXIT_DIVERGED, 0, 3)): + run_out = os.path.join(work, f"summary_{mode}") + done = _run(["--manifest", good, "--out", run_out, "--quiet"], + engine=engine, mode=mode) + if done.returncode != code: + fails.append(("compare-summary", + f"a manifest run with a {mode} double exited " + f"{done.returncode}, expected {code}. " + f"{done.stderr[:300]}")) + path = os.path.join(run_out, "summary.json") + if not os.path.exists(path): + fails.append(("compare-summary", + f"a {mode} manifest run wrote no summary.json — the " + f"run summary is the record, and it is written " + f"whether or not the run agreed")) + continue + summary = json.loads(_read(run_out, "summary.json")) + totals = summary.get("totals", {}) + want = {"documents_extracted": 3, "compare_attempted": 3, + "agreed": agreed, "diverged": diverged} + for field, value in want.items(): + if totals.get(field) != value: + fails.append(("compare-summary", + f"{mode}: totals[{field!r}] is " + f"{totals.get(field)!r}, expected {value}")) + rows = {row["target"]: row for row in summary.get("targets", [])} + if sorted(rows) != ["alpha", "beta"]: + fails.append(("compare-summary", + f"{mode}: the summary names targets {sorted(rows)}, " + f"expected the two the manifest declares")) + elif (rows["alpha"]["compare_attempted"] != 2 + or rows["beta"]["compare_attempted"] != 1): + fails.append(("compare-summary", + f"{mode}: the per-target denominators are not the " + f"manifest's ({rows['alpha']}, {rows['beta']})")) + if summary.get("engine_binary_sha256") != engine_digest: + fails.append(("compare-summary", + f"{mode}: the summary names adapter digest " + f"{summary.get('engine_binary_sha256')}, expected " + f"{engine_digest}")) + results = [n for n in sorted(os.listdir(run_out)) + if n.endswith(".result.json")] + if len(results) != 3: + fails.append(("compare-summary", + f"{mode}: {len(results)} per-document result(s) " + f"written, expected 3 — a manifest run IS the " + f"record, so every document's result is written")) + + # 6. A DECLARED target the run never reached fails it. Without the + # declaration such a target is simply absent from the summary, which + # is the shape a skipped repository would have. + skipped_out = os.path.join(work, "skipped") + skipped = manifest("skipped", documents, + targets=["alpha", "beta", "gamma"]) + done = _run(["--manifest", skipped, "--out", skipped_out, "--quiet"], + engine=engine, mode="faithful") + blob = done.stdout + done.stderr + if done.returncode != EXIT_USAGE: + fails.append(("compare-skipped-target", + f"a declared target with no documents exited " + f"{done.returncode}, expected {EXIT_USAGE}: every " + f"document that DID run agreed, and the run still is " + f"not evidence about 'gamma'")) + if "gamma" not in blob: + fails.append(("compare-skipped-target", + f"the driver did not name the target it never reached. " + f"Output: {blob[:400]}")) + return fails + + def _result_of(done: subprocess.CompletedProcess[str], label: str, fails: list[tuple[str, str]]) -> dict[str, object] | None: """The driver's JSON result, or a reported failure. @@ -370,6 +622,7 @@ def _adapter_controls(adapter: str) -> list[tuple[str, str]]: def run() -> int: fails: list[tuple[str, str]] = [] fails += _double_controls() + fails += _manifest_controls() adapter = _adapter() required = os.environ.get(REQUIRED_ENV) == "1" if adapter is None: @@ -397,7 +650,11 @@ def run() -> int: print(f"shadow compare controls OK: 10 double-driven controls held (crash, " f"timeout, garbage, protocol skew, three attestation traps, the " f"one-read invariant, an input disagreement and a renderer-only " - f"divergence)" + f"divergence), 6 manifest/identity controls (the adapter named by the " + f"digest of the file that ran, in a result and in a failure report; an " + f"empty run; a stale facts_sha256 refused before any engine; the " + f"summary counts derived from the outcomes; a declared target nothing " + f"reached)" f"{through_adapter}") return 0 diff --git a/tests/test_shadow_sweep.py b/tests/test_shadow_sweep.py new file mode 100644 index 00000000..ef3fd203 --- /dev/null +++ b/tests/test_shadow_sweep.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +"""The sweep interpreter's own controls (P-022 step 7a, #260 final acceptance). + +`tests/shadow_sweep.py` is what decides whether a recorded sweep run is +EVIDENCE. It is therefore the thing that must not be able to degrade into +"read the file and print OK": every rule it states gets a pair here that breaks +exactly that rule, and is reported for it. + +The pairs are synthetic on purpose. A control built from the committed run +would agree with it by construction — including on the day the committed run is +wrong — and it could not construct the failures at all, because a recorded run +that skipped a target is precisely the file that must never exist in the tree. + +`provenance_problems` is exercised separately from the rest: it is the one rule +whose answer depends on the checkout rather than on the pair, so a control that +folded it in would be reporting on git rather than on the interpreter. + +Run: python tests/test_shadow_sweep.py + python tests/run_tests.py (auto-discovered with every other test_*.py) +""" + +from __future__ import annotations + +import copy +import os +import sys +from typing import Any + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from shadow_sweep import TARGET_FIELDS, compute_sweep_summary + +PIN = "a" * 40 +RAW = "b" * 64 +CANON = "c" * 64 +ADAPTER = "d" * 64 +COMMAND = "scripts/own-check.sh --emit-facts -- targets/T" + + +def _definition() -> dict[str, Any]: + return { + "schema": 1, + "sweep": "p022-shadow-sweep", + "driver_version": 2, + "targets": [{"target": "T", "pinned_commit": PIN}], + "documents": [{ + "id": "T.repo", "target": "T", "target_commit": PIN, + "extraction_mode": "directory-walk", "extraction_command": COMMAND, + "timeout_seconds": 600.0, + }], + } + + +def _target_row(**over: int) -> dict[str, Any]: + row: dict[str, Any] = {"target": "T", **dict.fromkeys(TARGET_FIELDS, 0)} + row.update({"documents_extracted": 1, "compare_attempted": 1, "agreed": 1}) + row.update(over) + return row + + +def _result(source_commit: str) -> dict[str, Any]: + row = _target_row() + return { + "schema": 1, + "sweep": "p022-shadow-sweep", + "source_commit": source_commit, + "recorded_at": "2026-09-07T00:00:00Z", + "host": "test", + "workflow_run_url": None, + "driver_version": 2, + "adapters": [{"sha256": ADAPTER, "bytes": 1024}], + "documents": [{ + "id": "T.repo", "target": "T", "target_commit": PIN, + "extraction_mode": "directory-walk", "extraction_command": COMMAND, + "facts_sha256": RAW, + "raw": {"algorithm": "sha256", "digest": RAW, "bytes": 10}, + "canonical": {"algorithm": "sha256", "digest": CANON, "bytes": 10}, + "outcome": "agreed", "timeout_seconds": 600.0, + "reduction_outcome": "identical", + "declared_boundary_observations": 0, + "acceptance_unexplained_observations": 0, + "derived_outcome": "equal", "wall_clock_seconds": 0.1, + }], + "targets": [row], + "totals": {name: int(row[name]) for name in TARGET_FIELDS}, + } + + +def _head() -> str: + """A commit this checkout really contains, so that the POSITIVE control is + not silently failing on provenance and passing on everything else.""" + import subprocess + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + try: + done = subprocess.run(["git", "-C", root, "rev-parse", "HEAD"], + capture_output=True, text=True, check=False, + timeout=60) + except (OSError, subprocess.SubprocessError): + return "" + return done.stdout.strip() if done.returncode == 0 else "" + + +def run() -> int: + fails: list[str] = [] + head = _head() + + def problems(mutate: Any) -> list[str]: + definition, result = _definition(), _result(head) + mutate(definition, result) + return compute_sweep_summary(definition, result).problems + + def expect(label: str, needle: str, mutate: Any) -> None: + found = problems(mutate) + if not any(needle in p for p in found): + fails.append(f"{label}: the interpreter did not report it " + f"(expected {needle!r}); it said {found}") + + # THE POSITIVE control, and it comes first: a pair with nothing wrong must + # produce NO problems. Without it every assertion below is satisfied by an + # interpreter that reports everything about everything. + if head: + clean = problems(lambda d, r: None) + if clean: + fails.append(f"a clean pair was reported as having problems: {clean}") + else: + print("shadow sweep interpreter: the positive control is SKIPPED — this " + "is not a git checkout, so no commit can play the part of a valid " + "provenance") + + # 1. #250's fifth failure mode, in both of its shapes. + expect("an empty run", "compared zero of them", + lambda d, r: (r.update(documents=[], targets=[], + totals=dict.fromkeys(TARGET_FIELDS, 0)))) + expect("a target with nothing compared", "ZERO documents compared", + lambda d, r: (r["targets"].__setitem__( + 0, _target_row(compare_attempted=0, agreed=0)), + r["totals"].update(compare_attempted=0, agreed=0), + r.update(documents=[]))) + + # 2. The denominator is the DEFINITION's, in both directions. + expect("a declared document the run does not carry", + "absent from the run", + lambda d, r: d["documents"].append( + {**d["documents"][0], "id": "T.sln"})) + expect("a measured document the definition does not declare", + "not declared by the definition", + lambda d, r: r["documents"].append( + {**r["documents"][0], "id": "T.sln"})) + expect("a declared target missing from the run", + "a skipped target is not a passed repository", + lambda d, r: d["targets"].append({"target": "U", + "pinned_commit": PIN})) + + # 3. A run that measured SOMETHING, but not this. + expect("a document measured at another commit", "target_commit=", + lambda d, r: r["documents"][0].update(target_commit="f" * 40)) + expect("a document measured in another mode", "extraction_mode=", + lambda d, r: r["documents"][0].update(extraction_mode="solution")) + expect("a document measured by another command", "extraction_command=", + lambda d, r: r["documents"][0].update(extraction_command="cat")) + + # 4. The outcome, and the outcome hiding inside an outcome. + expect("a document that did not agree", "Only 'agreed' is an outcome", + lambda d, r: r["documents"][0].update(outcome="diverged")) + expect("an unexplained observation inside an agreed document", + "acceptance-unexplained observation(s) in a document recorded as", + lambda d, r: r["documents"][0].update( + acceptance_unexplained_observations=1)) + + # 5. The engine and the driver the run cannot name. + expect("a run that names no adapter", "names no adapter at all", + lambda d, r: r.update(adapters=[])) + expect("an adapter named by something that is not a digest", + "a path is not an identity", + lambda d, r: r.update(adapters=[{"sha256": "release", "bytes": 0}])) + expect("a driver version the definition does not expect", + "the definition expects", + lambda d, r: r.update(driver_version=1)) + expect("a document with no raw byte identity", "no raw byte identity", + lambda d, r: r["documents"][0].update(raw={})) + + # 6. A total that is not the sum of its parts is a number somebody typed. + expect("totals that disagree with the per-target rows", + "is a number somebody typed", + lambda d, r: r["totals"].update(agreed=7)) + + # 7. Provenance, on its own: a commit this checkout does not contain. + if head: + definition, result = _definition(), _result("0" * 40) + found = compute_sweep_summary(definition, result).problems + if not any("does not exist in this repository" in p for p in found): + fails.append(f"a run naming a commit this repository does not " + f"contain was accepted: {found}") + + # 8. The positive control must not be an accident of a shallow definition: + # a SECOND document, declared and measured, must also come back clean. + if head: + definition, result = _definition(), _result(head) + second_def = copy.deepcopy(definition["documents"][0]) + second_res = copy.deepcopy(result["documents"][0]) + second_def["id"] = second_res["id"] = "T.sln" + second_def["extraction_mode"] = second_res["extraction_mode"] = "solution" + definition["documents"].append(second_def) + result["documents"].append(second_res) + result["targets"][0]["documents_extracted"] = 2 + result["targets"][0]["compare_attempted"] = 2 + result["targets"][0]["agreed"] = 2 + result["totals"]["documents_extracted"] = 2 + result["totals"]["compare_attempted"] = 2 + result["totals"]["agreed"] = 2 + found = compute_sweep_summary(definition, result).problems + if found: + fails.append(f"a clean two-document pair was reported as having " + f"problems: {found}") + + if fails: + for f in fails: + print(f"FAIL[shadow-sweep-interpreter]: {f}") + return 1 + print("shadow sweep interpreter OK: 16 controls held (the empty run and the " + "target with nothing compared, the denominator in both directions, a " + "document measured at another commit / mode / command, a divergence " + "and an unexplained observation hidden inside an agreed document, an " + "unnameable adapter and driver, a missing raw identity, typed totals, " + "a foreign provenance, and two positive controls)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(run())