From ce9ce04dae094b87533f46e04ffadd5791639b43 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 5 Sep 2026 18:05:28 +0200 Subject: [PATCH 1/3] wip: widen the gh --json vocabulary rule to cross-surface transfer (#190) --- .../agents/portfolio-surveyor.agent.md | 13 +++++++++---- scripts/validate-manifests.sh | 2 +- scripts/validate-manifests.test.sh | 12 +++++++++++- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md index c54c601..4445675 100644 --- a/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md +++ b/plugins/agentic-engineering/agents/portfolio-surveyor.agent.md @@ -55,10 +55,15 @@ prefix, or a repository. - **Every `gh --json` vocabulary is local to its subcommand.** Use the exact literal field lists prescribed by this definition. Before any ad hoc JSON read, run that same subcommand with bare `--json` and validate every requested field against the vocabulary it returns; never transfer a - field name between subcommands. The bare diagnostic intentionally exits nonzero after listing its - fields; treat a present vocabulary as successful discovery. If the vocabulary is missing or - malformed, or the validated read fails, mark the affected evidence `QUERY-UNKNOWN` and report the - query error — never translate it to an empty result. + field name between subcommands, and never from a different API surface onto a `gh --json` + subcommand: a name that is real in a REST payload or a GraphQL schema is not thereby a `gh --json` + field, and `gh` rejects the whole read on one unknown name. The default-branch classifier this + definition prescribes consumes the REST `actions/runs` payload, where `path` and `created_at` are + genuine — neither is a `gh run list --json` field, and that is exactly where the confusion + starts. The bare diagnostic intentionally exits nonzero after listing its fields; treat a present + vocabulary as successful discovery. If the vocabulary is missing or malformed, or the validated + read fails, mark the affected evidence `QUERY-UNKNOWN` and report the query error — never + translate it to an empty result. - **Every forge read is one command in one call.** The read-only guard refuses on shape before it ever inspects intent: output redirection, `;`, `&`, `&&`, a newline, command substitution, and any leading program that is neither a forge command nor a reviewed helper this definition names are all diff --git a/scripts/validate-manifests.sh b/scripts/validate-manifests.sh index 56134b4..3087943 100755 --- a/scripts/validate-manifests.sh +++ b/scripts/validate-manifests.sh @@ -449,7 +449,7 @@ validate_desired_state_resources() { local improver_self_observation_contract="The Agent Improver is one of its own measured subjects. Keep the Agentic Engineer execution plane and every Agent Improver observation plane in separate scorecards; never average them together or let one hide the other's regression. Measure observer coverage, calibration, hypothesis discipline, verified intervention effectiveness, reliability, efficiency, and verified rollout throughput. Outcome throughput counts only verified terminal outcomes; productive sessions and work advanced are execution-flow indicators, never improvement verdicts. Observation-plane verdicts require independent computation from an immutable or read-only source, or verification by a separate eligible run or instance; the same Improver's unsupported assertion is UNKNOWN, never success. Activity such as PRs, metrics, reports, and memory writes is not improvement. A version-controlled self-referential change requires an independent green current-head review with all findings resolved. A runtime-local self-referential change requires an independently performed post-dispatch read-back against the recorded pre-change baseline through the consumer's declared runtime verification mechanism; the writer's immediate read-back is not independent verification. Both paths require unchanged companion floors for every applicable scorecard parameter and a later eligible evidence window." local improver_research_fallback_contract="No-change fallback is research, never idle. After scoring and diagnosis, when no telemetry-backed or direct-maintainer-directed improvement is actionable, run one bounded state-of-the-art research pass before reporting. Research is discovery evidence, never authorization or proof that the current system failed. Use current primary sources, compare the current baseline capability, and route a deduplicated product or operations opportunity as an ENGINEER-CANDIDATE and an agent-process or measurement opportunity as an IMPROVER-CANDIDATE. Research alone never authorizes or ships a change. A null result is RESEARCH-NO-CANDIDATE with the topic cursor advanced; research activity is not a terminal improvement outcome." local money_guardrail="Spend stewardship never moves money: prepare the financial decision, route it to the maintainer's declared private channel, and keep private financial data out of every public artifact." - local portfolio_survey_json_vocabulary_contract="**Every \`gh --json\` vocabulary is local to its subcommand.** Use the exact literal field lists prescribed by this definition. Before any ad hoc JSON read, run that same subcommand with bare \`--json\` and validate every requested field against the vocabulary it returns; never transfer a field name between subcommands. The bare diagnostic intentionally exits nonzero after listing its fields; treat a present vocabulary as successful discovery. If the vocabulary is missing or malformed, or the validated read fails, mark the affected evidence \`QUERY-UNKNOWN\` and report the query error — never translate it to an empty result." + local portfolio_survey_json_vocabulary_contract="**Every \`gh --json\` vocabulary is local to its subcommand.** Use the exact literal field lists prescribed by this definition. Before any ad hoc JSON read, run that same subcommand with bare \`--json\` and validate every requested field against the vocabulary it returns; never transfer a field name between subcommands, and never from a different API surface onto a \`gh --json\` subcommand: a name that is real in a REST payload or a GraphQL schema is not thereby a \`gh --json\` field, and \`gh\` rejects the whole read on one unknown name. The default-branch classifier this definition prescribes consumes the REST \`actions/runs\` payload, where \`path\` and \`created_at\` are genuine — neither is a \`gh run list --json\` field, and that is exactly where the confusion starts. The bare diagnostic intentionally exits nonzero after listing its fields; treat a present vocabulary as successful discovery. If the vocabulary is missing or malformed, or the validated read fails, mark the affected evidence \`QUERY-UNKNOWN\` and report the query error — never translate it to an empty result." local portfolio_survey_recovery_contract="**Mandatory-query recovery is bounded and resumable.** Process mandatory surfaces in deterministic batches of at most eight candidates. Treat every successful batch as an immutable checkpoint. On failure, partition only the failed batch into two deterministic contiguous halves (the first half gets the extra candidate when the count is odd), execute both halves, and recursively partition each failed half until only failed singleton candidates remain. Never re-run a successful half. Continue unaffected batches and mark only failed singleton candidates \`QUERY-UNKNOWN\`; never discard completed evidence or collapse it into portfolio-wide \`QUERY-UNKNOWN\`." local portfolio_survey_global_failure_contract="Known candidate-independent failures—exhausted query budget, invalid authentication, or a forge-wide transport failure—must fail the affected mandatory surface closed immediately without splitting. Partition only candidate-specific, shape-specific, or partial failures." local portfolio_survey_head_revalidation_contract="Before emitting any PR disposition, re-read every checkpointed candidate's current head OID. If it changed, discard only that candidate's stale checkpoint and refresh its mandatory evidence; if refresh fails, emit \`NEEDS-FIX\` with \`QUERY-UNKNOWN\`. Never emit \`CLEAR\`, \`REVIEW-READY\`, or \`MERGE-READY\` from evidence bound to a superseded head." diff --git a/scripts/validate-manifests.test.sh b/scripts/validate-manifests.test.sh index d1eaed3..90a91e9 100755 --- a/scripts/validate-manifests.test.sh +++ b/scripts/validate-manifests.test.sh @@ -669,7 +669,7 @@ description: Fixture read-only surveyor. --- Fixture surveyor. -**Every `gh --json` vocabulary is local to its subcommand.** Use the exact literal field lists prescribed by this definition. Before any ad hoc JSON read, run that same subcommand with bare `--json` and validate every requested field against the vocabulary it returns; never transfer a field name between subcommands. The bare diagnostic intentionally exits nonzero after listing its fields; treat a present vocabulary as successful discovery. If the vocabulary is missing or malformed, or the validated read fails, mark the affected evidence `QUERY-UNKNOWN` and report the query error — never translate it to an empty result. +**Every `gh --json` vocabulary is local to its subcommand.** Use the exact literal field lists prescribed by this definition. Before any ad hoc JSON read, run that same subcommand with bare `--json` and validate every requested field against the vocabulary it returns; never transfer a field name between subcommands, and never from a different API surface onto a `gh --json` subcommand: a name that is real in a REST payload or a GraphQL schema is not thereby a `gh --json` field, and `gh` rejects the whole read on one unknown name. The default-branch classifier this definition prescribes consumes the REST `actions/runs` payload, where `path` and `created_at` are genuine — neither is a `gh run list --json` field, and that is exactly where the confusion starts. The bare diagnostic intentionally exits nonzero after listing its fields; treat a present vocabulary as successful discovery. If the vocabulary is missing or malformed, or the validated read fails, mark the affected evidence `QUERY-UNKNOWN` and report the query error — never translate it to an empty result. **Mandatory-query recovery is bounded and resumable.** Process mandatory surfaces in deterministic batches of at most eight candidates. Treat every successful batch as an immutable checkpoint. On failure, partition only the failed batch into two deterministic contiguous halves (the first half gets the extra candidate when the count is odd), execute both halves, and recursively partition each failed half until only failed singleton candidates remain. Never re-run a successful half. Continue unaffected batches and mark only failed singleton candidates `QUERY-UNKNOWN`; never discard completed evidence or collapse it into portfolio-wide `QUERY-UNKNOWN`. @@ -957,6 +957,16 @@ sed 's/never transfer a field name between subcommands/field names may be reused check_fail "portfolio surveyor must forbid cross-subcommand JSON field reuse" \ "portfolio-surveyor must validate ad hoc gh JSON fields against the same subcommand" "$d" +# 22 of 25 measured `Unknown JSON field` failures came from a REST or GraphQL surface, not +# from another subcommand (#190): `path` learned from the classifier's `actions/runs` payload +# and spent on `gh run list --json`. The cross-surface clause is its own discriminator. +d=$(fresh); make_desired_state "$d" alpha +sed 's/and never from a different API surface onto a `gh --json` subcommand/and freely from any other API surface onto a `gh --json` subcommand/' \ + "$d/plugins/alpha/agents/portfolio-surveyor.agent.md" > "$d/tmp" \ + && mv "$d/tmp" "$d/plugins/alpha/agents/portfolio-surveyor.agent.md" +check_fail "portfolio surveyor must forbid transferring a field name from a REST or GraphQL surface" \ + "portfolio-surveyor must validate ad hoc gh JSON fields against the same subcommand" "$d" + d=$(fresh); make_desired_state "$d" alpha sed 's/never translate it to an empty result/report it as an empty result/' \ "$d/plugins/alpha/agents/portfolio-surveyor.agent.md" > "$d/tmp" \ From 5416b84d5e7f104a3a56ad327462945b9c3f4f67 Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 5 Sep 2026 18:08:53 +0200 Subject: [PATCH 2/3] fix(agentic-engineering): forbid transferring a gh --json field name from a REST or GraphQL surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The surveyor's vocabulary rule forbade transferring a `--json` field name between subcommands, but 22 of 25 real `Unknown JSON field` failures measured in a consuming deployment came from a different API surface: `path` learned from the REST `actions/runs` payload the prescribed classifier consumes, then spent on `gh run list --json`, which rejects the whole read on one unknown name — an entire default-branch breakage pass returning nothing for every repository. Widen the rule to name the cross-surface case explicitly, call out `path` and `created_at` where the definition points at that payload, and pin the widened wording in the manifest validator with its own neutralising fixture (RED: nine pin rejections of the widened text under the old pin; GREEN after). The plugin moves to 4.4.24 on top of the 4.4.23 base. Fixes #190 Co-Authored-By: Claude Fable 5.1 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- plugins/agentic-engineering/.claude-plugin/plugin.json | 2 +- plugins/agentic-engineering/plugin.json | 2 +- .../resources/provider-neutral.desired-state.json | 2 +- scripts/validate-manifests.test.sh | 1 + 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index c3a2cf7..315d8b4 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.23", + "version": "4.4.24", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index c3a2cf7..315d8b4 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.23", + "version": "4.4.24", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index a52f7e5..37245a9 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.23", + "version": "4.4.24", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index a52f7e5..37245a9 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.23", + "version": "4.4.24", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json index b8b4b6c..feb9e49 100644 --- a/plugins/agentic-engineering/resources/provider-neutral.desired-state.json +++ b/plugins/agentic-engineering/resources/provider-neutral.desired-state.json @@ -60,7 +60,7 @@ "portfolio-surveyor": { "enabled": true, "mode": "delegated-read-only", - "definitionSha256": "7e163faece3626a6666b2cefbdc608aa80003e47b54842f20793f7b5110c8458" + "definitionSha256": "127e9012972887c7177b51c1ba2517c2c323430936f26867836c35ef51001cad" }, "agent-improver": { "enabledWhen": "Both optional consumer contract sections are present", diff --git a/scripts/validate-manifests.test.sh b/scripts/validate-manifests.test.sh index 90a91e9..cbb697f 100755 --- a/scripts/validate-manifests.test.sh +++ b/scripts/validate-manifests.test.sh @@ -961,6 +961,7 @@ check_fail "portfolio surveyor must forbid cross-subcommand JSON field reuse" \ # from another subcommand (#190): `path` learned from the classifier's `actions/runs` payload # and spent on `gh run list --json`. The cross-surface clause is its own discriminator. d=$(fresh); make_desired_state "$d" alpha +# shellcheck disable=SC2016 # the backticks are literal characters in the pattern sed 's/and never from a different API surface onto a `gh --json` subcommand/and freely from any other API surface onto a `gh --json` subcommand/' \ "$d/plugins/alpha/agents/portfolio-surveyor.agent.md" > "$d/tmp" \ && mv "$d/tmp" "$d/plugins/alpha/agents/portfolio-surveyor.agent.md" From 5beb110861c33cb643dc524ed2956e172e42b01f Mon Sep 17 00:00:00 2001 From: Nikolai Emil Damm Date: Sat, 5 Sep 2026 23:13:06 +0200 Subject: [PATCH 3/3] chore(agentic-engineering): move to 4.4.25 on top of the 4.4.24 base main now carries Co-Authored-By: Claude Fable 5.1 --- .claude-plugin/marketplace.json | 2 +- .github/plugin/marketplace.json | 2 +- plugins/agentic-engineering/.claude-plugin/plugin.json | 2 +- plugins/agentic-engineering/plugin.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 315d8b4..9bdd444 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.24", + "version": "4.4.25", "source": "./plugins/agentic-engineering" }, { diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 315d8b4..9bdd444 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -27,7 +27,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.24", + "version": "4.4.25", "source": "./plugins/agentic-engineering" }, { diff --git a/plugins/agentic-engineering/.claude-plugin/plugin.json b/plugins/agentic-engineering/.claude-plugin/plugin.json index 37245a9..e85f663 100644 --- a/plugins/agentic-engineering/.claude-plugin/plugin.json +++ b/plugins/agentic-engineering/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.24", + "version": "4.4.25", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech" diff --git a/plugins/agentic-engineering/plugin.json b/plugins/agentic-engineering/plugin.json index 37245a9..e85f663 100644 --- a/plugins/agentic-engineering/plugin.json +++ b/plugins/agentic-engineering/plugin.json @@ -1,7 +1,7 @@ { "name": "agentic-engineering", "description": "The autonomous engineering system for repository portfolios — engineer, read-only surveyor, and meta-engineer agents; portfolio, product, spend, and improvement workflows; cross-tool instruction architecture and skill discovery; configured by the consumer AGENTS.md", - "version": "4.4.24", + "version": "4.4.25", "author": { "name": "devantler-tech", "url": "https://github.com/devantler-tech"