diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4429b6d61..369de3984 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: - name: Validate agent skills and plugin packages run: | - bun test scripts/clawhub-ignore-policy.test.ts scripts/claude-mcp-config-policy.test.ts scripts/public-skill-discovery-policy.test.ts + bun test scripts/clawhub-ignore-policy.test.ts scripts/claude-mcp-config-policy.test.ts scripts/openai-tool-annotation-policy.test.ts scripts/public-skill-discovery-policy.test.ts bun scripts/validate-skills.ts - name: Type check diff --git a/packages/mcp/src/tools/read-tool-annotations.test.ts b/packages/mcp/src/tools/read-tool-annotations.test.ts index 6ef396a84..3e2ee2ccb 100644 --- a/packages/mcp/src/tools/read-tool-annotations.test.ts +++ b/packages/mcp/src/tools/read-tool-annotations.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'bun:test' -import { mkdtempSync, rmSync } from 'node:fs' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' -import { join } from 'node:path' +import { join, resolve } from 'node:path' import { Client } from '@modelcontextprotocol/sdk/client/index.js' import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js' import { SceneBridge } from '../bridge/scene-bridge' @@ -100,6 +100,19 @@ const TOOL_POLICIES = [ ] as const const EXPECTED_TOOL_NAMES = TOOL_POLICIES.flatMap(({ tools }) => tools).toSorted() +const annotationPacket = JSON.parse( + readFileSync( + resolve(import.meta.dir, '../../../../plugin-evals/tool-annotation-justifications.json'), + 'utf8', + ), +) as { + required_hints: Array<'readOnlyHint' | 'destructiveHint' | 'openWorldHint'> + tools: Array<{ + name: string + annotations: Record<'readOnlyHint' | 'destructiveHint' | 'openWorldHint', boolean> + justifications: Record<'readOnlyHint' | 'destructiveHint' | 'openWorldHint', string> + }> +} describe('MCP tool annotations', () => { test('classifies every registered tool for approval-aware clients', async () => { @@ -123,6 +136,15 @@ describe('MCP tool annotations', () => { expect(byName.get(name)?.annotations).toEqual(policy.annotations) } } + + expect(annotationPacket.tools.map(({ name }) => name)).toEqual(EXPECTED_TOOL_NAMES) + for (const tool of annotationPacket.tools) { + const registeredAnnotations = byName.get(tool.name)?.annotations + for (const hint of annotationPacket.required_hints) { + expect(tool.annotations[hint]).toBe(registeredAnnotations?.[hint]) + expect(tool.justifications[hint].trim().length).toBeGreaterThan(0) + } + } } finally { await client.close() await server.close() diff --git a/plugin-evals/README.md b/plugin-evals/README.md index 001219049..bc405e697 100644 --- a/plugin-evals/README.md +++ b/plugin-evals/README.md @@ -1,8 +1,10 @@ # Publishing evaluation fixtures -`publishing-cases.json` is the draft cross-skill review suite for a future OpenAI **With MCP** submission. It contains at least five positive cases with expected result shapes, plus three negative or refusal-boundary cases with explicit reasons the plugin must not complete the requested action. The suite is blocked until Pascal provisions a stable hosted MCP review environment, OAuth-compatible reviewer access, and named disposable fixtures that reviewers can use without internal context. Each standalone skill also bundles: +`publishing-cases.json` is the draft cross-skill review suite for a future OpenAI **With MCP** submission. It contains at least five positive cases with expected result shapes, plus three negative or refusal-boundary cases with explicit reasons the plugin must not complete the requested action. `tool-annotation-justifications.json` records the exact three required hint values and a non-empty justification for every hint on all 46 expected MCP tools. Repository policy tests check its exact inventory and shape, and the live MCP regression test checks its values against `tools/list`. + +The suite remains blocked until Pascal confirms an authorized verified OpenAI publisher identity, completes portal-token domain verification, passes Scan Tools against the production hosted endpoint, provisions OAuth-compatible reviewer access, and names disposable fixtures that reviewers can use without internal context. Each standalone skill also bundles: - `evals/evals.json` for task behavior; - `evals/trigger-evals.json` for description routing, with at least five positive and three negative queries. -The shared suite lives outside `skills/` because it is release evidence rather than skill runtime content. These draft cases are not reproducible reviewer materials yet, and their presence does not mean a native host or OpenAI portal scan has passed them. Record those results separately after the blocked hosted-MCP prerequisites exist. +The shared suite and annotation packet live outside `skills/` because they are submission evidence rather than skill runtime content. This repository state is preparation only: the draft cases are not reproducible reviewer materials yet, and no portal scan, domain verification, publisher verification, submission, approval, or publication is represented. Record those results separately after the blocked hosted-MCP prerequisites exist. diff --git a/plugin-evals/publishing-cases.json b/plugin-evals/publishing-cases.json index 7612059d4..4e6b88e3b 100644 --- a/plugin-evals/publishing-cases.json +++ b/plugin-evals/publishing-cases.json @@ -3,20 +3,25 @@ "submission_route": "with_mcp", "status": "blocked", "blockers": [ + "A verified OpenAI publisher or developer identity authorized to submit for Pascal has not been confirmed.", + "Pascal's submission domain has not been verified with the portal-generated /.well-known/openai-apps-challenge token.", "Production hosted MCP endpoint has not passed OpenAI Scan Tools for this candidate.", "OAuth-compatible reviewer access and reusable demo credentials without MFA, SMS, email confirmation, or private-network access are not prepared.", "Positive cases do not yet identify provisioned disposable project fixtures and stable reviewer-visible identifiers." ], "tool_annotation_validation": { "status": "local_pass", - "checked_at": "2026-09-09", + "checked_at": "2026-09-10", "registered_tools": 46, "required_hints": ["readOnlyHint", "destructiveHint", "openWorldHint"], + "justification_packet": "plugin-evals/tool-annotation-justifications.json", "evidence": [ - "The live MCP tools/list enumeration test classified all 46 registered tools and failed on any unlisted or unannotated tool.", - "The complete @pascal-app/mcp suite passed 361 tests with 1407 assertions, and the package TypeScript build passed." + "The machine-readable packet records exact values and non-empty per-hint justifications for all 46 expected tools.", + "The repository policy validator rejects missing, unexpected, duplicate, reordered, or wrongly classified tools and missing, blank, or extra hint justifications.", + "The live MCP tools/list regression test requires the packet's exact inventory and annotation values to match the registered server tools.", + "The complete @pascal-app/mcp suite passed 361 tests with 1684 assertions, and the MCP dependency build passed." ], - "limitations": "Local enumeration and package checks do not establish production hosted-MCP behavior, OpenAI Scan Tools approval, reviewer access, submission, or listing." + "limitations": "This is local submission preparation only. It does not establish production hosted-MCP behavior, a portal Scan Tools run or approval, domain verification, verified publisher identity, reviewer access, submission, review, publication, or listing." }, "cases": [ { diff --git a/plugin-evals/release-notes.md b/plugin-evals/release-notes.md index 1e0a417c2..04b9546f7 100644 --- a/plugin-evals/release-notes.md +++ b/plugin-evals/release-notes.md @@ -4,4 +4,6 @@ Draft release notes for a future Pascal agent skills 0.1.7 **With MCP** submissi The plugin teaches ChatGPT and Codex to create, inspect, edit, validate, save, and hand off editable Pascal 3D scenes through a separately connected Pascal MCP server. It also includes a focused furniture-fit workflow that reports measured footprint evidence, unsupported checks, and one bounded next action without authorizing project changes or spending. -The source package contains two skills, portable Agent Plugins metadata, OpenAI listing metadata, and bundled square icons. Both skills require Pascal MCP tools for their tool-backed workflows, so they must not be submitted through the Skills only route. The With MCP submission remains blocked until the production endpoint passes Scan Tools, OAuth-compatible reviewer access and credentials exist, domain verification succeeds, and each positive case names a disposable fixture a reviewer can access without internal setup. No portal submission, scan, review, or publication is represented by this file. +The source package contains two skills, portable Agent Plugins metadata, OpenAI listing metadata, bundled square icons, and a machine-readable packet that gives exact values and per-hint justifications for all 46 expected MCP tools. Local validators require that packet to match the registered server inventory and annotation values. Both skills require Pascal MCP tools for their tool-backed workflows, so they must not be submitted through the Skills only route. + +The With MCP submission remains blocked until an authorized verified OpenAI publisher identity is confirmed, the portal-generated domain challenge is served and accepted, the production endpoint passes Scan Tools, OAuth-compatible reviewer access and credentials exist, and each positive case names a disposable fixture a reviewer can access without internal setup. This is submission preparation only. No portal scan, domain verification, submission, review, approval, publication, or listing is represented by this file. diff --git a/plugin-evals/tool-annotation-justifications.json b/plugin-evals/tool-annotation-justifications.json new file mode 100644 index 000000000..edb74fcde --- /dev/null +++ b/plugin-evals/tool-annotation-justifications.json @@ -0,0 +1,608 @@ +{ + "schema_version": 1, + "required_hints": [ + "readOnlyHint", + "destructiveHint", + "openWorldHint" + ], + "tools": [ + { + "name": "add_door", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds a door node to an existing wall in the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "add_window", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds a window node to an existing wall in the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "analyze_floorplan_image", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": true + }, + "justifications": { + "readOnlyHint": "It analyzes the supplied floor-plan image and returns observations without changing Pascal scene or project state.", + "destructiveHint": "It returns analysis results without creating, updating, replacing, or deleting Pascal scene or project state.", + "openWorldHint": "It may send the supplied image to an externally hosted vision model or provider to produce the analysis." + } + }, + { + "name": "analyze_room_photo", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": true + }, + "justifications": { + "readOnlyHint": "It analyzes the supplied room photo and returns observations without changing Pascal scene or project state.", + "destructiveHint": "It returns analysis results without creating, updating, replacing, or deleting Pascal scene or project state.", + "openWorldHint": "It may send the supplied image to an externally hosted vision model or provider to produce the analysis." + } + }, + { + "name": "apply_patch", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because applies arbitrary scene patch operations that can update or remove existing Pascal nodes.", + "destructiveHint": "A patch can update or delete existing nodes, so user-authored scene state can be overwritten or removed.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "check_collisions", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It computes collision and clearance results from supplied candidate geometry or the connected Pascal scene without changing that scene.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "create_from_template", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because instantiates a template into the active scene and can replace existing scene structure before saving.", + "destructiveHint": "Instantiating the template replaces the active scene graph, so existing active-scene content can be overwritten.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "create_house_from_brief", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because builds a house scene from a brief and can replace broad existing scene structure before saving.", + "destructiveHint": "Building from the brief replaces broad active-scene structure, so existing active-scene content can be overwritten.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "create_level", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because creates a new level in the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "create_project", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because creates a new Pascal project and its initial scene state.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "create_roof", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds a roof node to the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "create_room", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds room geometry to the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "create_stair_between_levels", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because creates a stair and changes related level geometry, including openings in existing slabs.", + "destructiveHint": "The operation updates existing slabs or ceilings with openings, so existing modeled geometry is changed.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "create_story_shell", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds a story shell to the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "create_wall", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds a wall node to the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "cut_opening", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds an opening definition to existing Pascal scene geometry.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "delete_node", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because deletes an existing node and may remove dependent scene content.", + "destructiveHint": "The operation removes an existing node and can cascade to its descendants.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "delete_scene", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because permanently deletes a persisted Pascal scene.", + "destructiveHint": "The operation permanently removes a persisted scene and its stored revisions.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "describe_node", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It reads and summarizes one node from the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "duplicate_level", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because creates a new level by copying an existing level and its contents.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "export_glb", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It serializes the connected Pascal scene as GLB output without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "export_json", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It serializes the connected Pascal scene as JSON output without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "find_nodes", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It searches nodes already present in the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "furnish_room", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds generated furniture placements to a room in the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "generate_variants", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because creates additional scene variants from the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "get_level_summary", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It reads a summary of one level in the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "get_node", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It reads one node from the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "get_project_status", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because can bind or switch the active project and load its scene, changing the connected session state.", + "destructiveHint": "When the requested project is not active, the operation loads its graph and replaces the active in-memory scene and session binding.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "get_scene", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It reads the active Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "get_walls", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It reads wall geometry from the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "get_zones", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It reads zone data from the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "list_levels", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It lists levels already present in the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "list_scenes", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It lists scenes available from the connected Pascal service without changing or selecting a scene.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "list_templates", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It lists built-in Pascal scene templates without instantiating or changing a scene.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "load_scene", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because replaces the active in-memory scene with the selected persisted scene.", + "destructiveHint": "Loading a persisted scene replaces the active in-memory scene and its current unsaved state.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "measure", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It computes measurements from nodes in the connected Pascal scene without changing scene or project state.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "photo_to_scene", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": true + }, + "justifications": { + "readOnlyHint": "It is not read-only because uses image analysis to generate and persist a new active Pascal scene, replacing prior active scene state.", + "destructiveHint": "It replaces active scene state and persists generated geometry, so existing user state can be overwritten.", + "openWorldHint": "It may send the supplied photo to an externally hosted vision model or provider before generating the scene." + } + }, + { + "name": "place_item", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because adds an item at the requested pose in the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "redo", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because reapplies a recorded change and mutates the active Pascal scene and history state.", + "destructiveHint": "Reapplying history changes the active scene and can overwrite state restored by undo.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "rename_scene", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because changes persisted scene metadata by replacing the existing scene name.", + "destructiveHint": "Renaming replaces persisted scene metadata and can overwrite the prior name.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "save_scene", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because writes the active scene to persistence and can replace the previously saved scene state.", + "destructiveHint": "Saving writes persisted state and can overwrite the previously saved scene revision.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "search_assets", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It searches Pascal asset catalog data available to the connected service without placing an asset or changing a scene.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "set_zone", + "annotations": { + "readOnlyHint": false, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because it creates a new zone in the connected Pascal scene.", + "destructiveHint": "Its intended operation adds new modeled state without deleting or replacing existing Pascal scene or project state.", + "openWorldHint": "It operates on the connected Pascal scene, project, and configured catalog data without accessing the public internet." + } + }, + { + "name": "undo", + "annotations": { + "readOnlyHint": false, + "destructiveHint": true, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It is not read-only because reverses a recorded change and mutates the active Pascal scene and history state.", + "destructiveHint": "Reversing history changes the active scene and can remove changes currently present in it.", + "openWorldHint": "It operates on the connected Pascal scene or project service without accessing the public internet." + } + }, + { + "name": "validate_scene", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It checks the connected Pascal scene for schema and graph problems without repairing or changing it.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + }, + { + "name": "verify_scene", + "annotations": { + "readOnlyHint": true, + "destructiveHint": false, + "openWorldHint": false + }, + "justifications": { + "readOnlyHint": "It computes verification results for the connected Pascal scene without repairing or changing it.", + "destructiveHint": "It does not create, update, replace, delete, load, or save Pascal scene or project state.", + "openWorldHint": "It operates only on data supplied by or already available inside the connected Pascal service and does not access the public internet." + } + } + ] +} diff --git a/scripts/openai-tool-annotation-policy.test.ts b/scripts/openai-tool-annotation-policy.test.ts new file mode 100644 index 000000000..aa6662c3f --- /dev/null +++ b/scripts/openai-tool-annotation-policy.test.ts @@ -0,0 +1,65 @@ +import { describe, expect, test } from 'bun:test' +import packet from '../plugin-evals/tool-annotation-justifications.json' +import { validateOpenAiToolAnnotationPacket } from './openai-tool-annotation-policy' + +const clonePacket = () => structuredClone(packet) + +describe('OpenAI tool annotation justification packet', () => { + test('accepts the canonical exact inventory', () => { + expect(validateOpenAiToolAnnotationPacket(packet)).toEqual([]) + }) + + test('rejects a missing tool', () => { + const candidate = clonePacket() + candidate.tools.pop() + expect(validateOpenAiToolAnnotationPacket(candidate)).toContain( + 'OpenAI tool annotation packet must contain the exact 46-tool inventory', + ) + }) + + test('rejects an unexpected or duplicate tool', () => { + const unexpected = clonePacket() + unexpected.tools[0]!.name = 'unexpected_tool' + expect(validateOpenAiToolAnnotationPacket(unexpected)).toContain( + 'OpenAI tool annotation packet contains unexpected tool unexpected_tool', + ) + + const duplicate = clonePacket() + duplicate.tools[1]!.name = duplicate.tools[0]!.name + expect(validateOpenAiToolAnnotationPacket(duplicate)).toContain( + 'OpenAI tool annotation packet tool names must be unique', + ) + }) + + test('rejects a wrong hint value', () => { + const candidate = clonePacket() + candidate.tools[0]!.annotations.readOnlyHint = true + expect(validateOpenAiToolAnnotationPacket(candidate)).toContain( + 'OpenAI tool annotation packet add_door has readOnlyHint=true, expected false', + ) + }) + + test('rejects missing, blank, or extra justifications', () => { + const missing = clonePacket() as unknown as { + tools: Array<{ justifications: Record }> + } + delete missing.tools[0].justifications.openWorldHint + expect(validateOpenAiToolAnnotationPacket(missing)).toContain( + 'OpenAI tool annotation packet add_door justifications must contain exactly the required hints', + ) + + const blank = clonePacket() + blank.tools[0]!.justifications.destructiveHint = ' ' + expect(validateOpenAiToolAnnotationPacket(blank)).toContain( + 'OpenAI tool annotation packet add_door needs a non-empty destructiveHint justification', + ) + + const extra = clonePacket() as unknown as { + tools: Array<{ justifications: Record }> + } + extra.tools[0].justifications.idempotentHint = 'Not part of this submission packet.' + expect(validateOpenAiToolAnnotationPacket(extra)).toContain( + 'OpenAI tool annotation packet add_door justifications must contain exactly the required hints', + ) + }) +}) diff --git a/scripts/openai-tool-annotation-policy.ts b/scripts/openai-tool-annotation-policy.ts new file mode 100644 index 000000000..01e01c990 --- /dev/null +++ b/scripts/openai-tool-annotation-policy.ts @@ -0,0 +1,191 @@ +export const OPENAI_REQUIRED_TOOL_HINTS = [ + 'readOnlyHint', + 'destructiveHint', + 'openWorldHint', +] as const + +type OpenAiToolHint = (typeof OPENAI_REQUIRED_TOOL_HINTS)[number] +type ToolAnnotations = Record + +const policy = ( + readOnlyHint: boolean, + destructiveHint: boolean, + openWorldHint: boolean, +): ToolAnnotations => ({ readOnlyHint, destructiveHint, openWorldHint }) + +export const EXPECTED_OPENAI_TOOL_ANNOTATIONS = { + add_door: policy(false, false, false), + add_window: policy(false, false, false), + analyze_floorplan_image: policy(true, false, true), + analyze_room_photo: policy(true, false, true), + apply_patch: policy(false, true, false), + check_collisions: policy(true, false, false), + create_from_template: policy(false, true, false), + create_house_from_brief: policy(false, true, false), + create_level: policy(false, false, false), + create_project: policy(false, false, false), + create_roof: policy(false, false, false), + create_room: policy(false, false, false), + create_stair_between_levels: policy(false, true, false), + create_story_shell: policy(false, false, false), + create_wall: policy(false, false, false), + cut_opening: policy(false, false, false), + delete_node: policy(false, true, false), + delete_scene: policy(false, true, false), + describe_node: policy(true, false, false), + duplicate_level: policy(false, false, false), + export_glb: policy(true, false, false), + export_json: policy(true, false, false), + find_nodes: policy(true, false, false), + furnish_room: policy(false, false, false), + generate_variants: policy(false, false, false), + get_level_summary: policy(true, false, false), + get_node: policy(true, false, false), + get_project_status: policy(false, true, false), + get_scene: policy(true, false, false), + get_walls: policy(true, false, false), + get_zones: policy(true, false, false), + list_levels: policy(true, false, false), + list_scenes: policy(true, false, false), + list_templates: policy(true, false, false), + load_scene: policy(false, true, false), + measure: policy(true, false, false), + photo_to_scene: policy(false, true, true), + place_item: policy(false, false, false), + redo: policy(false, true, false), + rename_scene: policy(false, true, false), + save_scene: policy(false, true, false), + search_assets: policy(true, false, false), + set_zone: policy(false, false, false), + undo: policy(false, true, false), + validate_scene: policy(true, false, false), + verify_scene: policy(true, false, false), +} as const satisfies Record + +const exactKeys = (value: Record, expected: readonly string[]): boolean => { + const actual = Object.keys(value).toSorted() + const sortedExpected = [...expected].toSorted() + return ( + actual.length === sortedExpected.length && + actual.every((key, index) => key === sortedExpected[index]) + ) +} + +export function validateOpenAiToolAnnotationPacket(value: unknown): string[] { + const failures: string[] = [] + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return ['OpenAI tool annotation packet must be an object'] + } + + const packet = value as Record + if (!exactKeys(packet, ['schema_version', 'required_hints', 'tools'])) { + failures.push( + 'OpenAI tool annotation packet must contain only schema_version, required_hints, and tools', + ) + } + if (packet.schema_version !== 1) + failures.push('OpenAI tool annotation packet schema_version must be 1') + if ( + !Array.isArray(packet.required_hints) || + packet.required_hints.length !== OPENAI_REQUIRED_TOOL_HINTS.length || + packet.required_hints.some((hint, index) => hint !== OPENAI_REQUIRED_TOOL_HINTS[index]) + ) { + failures.push( + `OpenAI tool annotation packet required_hints must be exactly ${OPENAI_REQUIRED_TOOL_HINTS.join(', ')}`, + ) + } + if (!Array.isArray(packet.tools)) { + failures.push('OpenAI tool annotation packet tools must be an array') + return failures + } + + const expectedNames = Object.keys(EXPECTED_OPENAI_TOOL_ANNOTATIONS).toSorted() + const names: string[] = [] + for (const [index, rawTool] of packet.tools.entries()) { + if (!rawTool || typeof rawTool !== 'object' || Array.isArray(rawTool)) { + failures.push(`OpenAI tool annotation entry ${index} must be an object`) + continue + } + const tool = rawTool as Record + if (!exactKeys(tool, ['name', 'annotations', 'justifications'])) { + failures.push( + `OpenAI tool annotation entry ${index} must contain only name, annotations, and justifications`, + ) + } + if (typeof tool.name !== 'string' || !tool.name) { + failures.push(`OpenAI tool annotation entry ${index} needs a non-empty name`) + continue + } + names.push(tool.name) + const expected = + EXPECTED_OPENAI_TOOL_ANNOTATIONS[tool.name as keyof typeof EXPECTED_OPENAI_TOOL_ANNOTATIONS] + if (!expected) { + failures.push(`OpenAI tool annotation packet contains unexpected tool ${tool.name}`) + continue + } + if ( + !tool.annotations || + typeof tool.annotations !== 'object' || + Array.isArray(tool.annotations) + ) { + failures.push(`OpenAI tool annotation packet ${tool.name} annotations must be an object`) + } else { + const annotations = tool.annotations as Record + if (!exactKeys(annotations, OPENAI_REQUIRED_TOOL_HINTS)) { + failures.push( + `OpenAI tool annotation packet ${tool.name} annotations must contain exactly the required hints`, + ) + } + for (const hint of OPENAI_REQUIRED_TOOL_HINTS) { + if (annotations[hint] !== expected[hint]) { + failures.push( + `OpenAI tool annotation packet ${tool.name} has ${hint}=${String(annotations[hint])}, expected ${String(expected[hint])}`, + ) + } + } + } + if ( + !tool.justifications || + typeof tool.justifications !== 'object' || + Array.isArray(tool.justifications) + ) { + failures.push(`OpenAI tool annotation packet ${tool.name} justifications must be an object`) + } else { + const justifications = tool.justifications as Record + if (!exactKeys(justifications, OPENAI_REQUIRED_TOOL_HINTS)) { + failures.push( + `OpenAI tool annotation packet ${tool.name} justifications must contain exactly the required hints`, + ) + } + for (const hint of OPENAI_REQUIRED_TOOL_HINTS) { + const justification = justifications[hint] + if (typeof justification !== 'string' || !justification.trim()) { + failures.push( + `OpenAI tool annotation packet ${tool.name} needs a non-empty ${hint} justification`, + ) + } else if (justification !== justification.trim()) { + failures.push( + `OpenAI tool annotation packet ${tool.name} ${hint} justification must not have surrounding whitespace`, + ) + } + } + } + } + + const sortedNames = [...names].toSorted() + if (new Set(names).size !== names.length) + failures.push('OpenAI tool annotation packet tool names must be unique') + if (names.some((name, index) => name !== sortedNames[index])) { + failures.push('OpenAI tool annotation packet tools must be sorted by name') + } + if ( + sortedNames.length !== expectedNames.length || + sortedNames.some((name, index) => name !== expectedNames[index]) + ) { + failures.push( + `OpenAI tool annotation packet must contain the exact ${expectedNames.length}-tool inventory`, + ) + } + + return failures +} diff --git a/scripts/validate-skills.ts b/scripts/validate-skills.ts index 6148ea594..8e7127f62 100644 --- a/scripts/validate-skills.ts +++ b/scripts/validate-skills.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from 'node:url' import { XMLParser, XMLValidator } from 'fast-xml-parser' import { validateClaudeMcpPolicy } from './claude-mcp-config-policy' import { validateClawHubIgnorePolicy } from './clawhub-ignore-policy' +import { validateOpenAiToolAnnotationPacket } from './openai-tool-annotation-policy' import { collectSkillDiscoveryEntries, validatePublicSkillDiscoverySurface, @@ -576,10 +577,21 @@ for (const [semanticCase, requirement] of requiredSemanticCases) { } const publishingFile = join(root, 'plugin-evals', 'publishing-cases.json') +const annotationPacketFile = join(root, 'plugin-evals', 'tool-annotation-justifications.json') +const annotationPacket = parseJson(annotationPacketFile) +for (const annotationFailure of validateOpenAiToolAnnotationPacket(annotationPacket)) { + fail(annotationFailure) +} const publishing = parseJson(publishingFile) as { submission_route?: unknown status?: unknown blockers?: unknown + tool_annotation_validation?: { + status?: unknown + registered_tools?: unknown + required_hints?: unknown + justification_packet?: unknown + } cases?: Array<{ id?: unknown skill?: unknown @@ -606,6 +618,16 @@ if ( ) { fail('Publishing suite must name its current hosted MCP review blockers') } +const annotationValidation = publishing.tool_annotation_validation +if ( + annotationValidation?.status !== 'local_pass' || + annotationValidation.registered_tools !== 46 || + JSON.stringify(annotationValidation.required_hints) !== + JSON.stringify(['readOnlyHint', 'destructiveHint', 'openWorldHint']) || + annotationValidation.justification_packet !== 'plugin-evals/tool-annotation-justifications.json' +) { + fail('Publishing suite must reference the locally validated exact 46-tool justification packet') +} const publishingCases = publishing.cases ?? [] let positivePublishingCases = 0 let negativePublishingCases = 0 diff --git a/skills/README.md b/skills/README.md index 4f748454f..1665b3b7e 100644 --- a/skills/README.md +++ b/skills/README.md @@ -81,9 +81,9 @@ The `source-reviewed` date records a code and public-documentation review. The ` ## Validate the source package ```bash -bun test scripts/clawhub-ignore-policy.test.ts scripts/claude-mcp-config-policy.test.ts scripts/public-skill-discovery-policy.test.ts +bun test scripts/clawhub-ignore-policy.test.ts scripts/claude-mcp-config-policy.test.ts scripts/openai-tool-annotation-policy.test.ts scripts/public-skill-discovery-policy.test.ts bun scripts/validate-skills.ts claude plugin validate . --strict ``` -The repository validator checks the exact two-skill public discovery surface, keeps contributor-only workflows internal, and checks frontmatter, bundled links, task and trigger fixtures, semantic furniture next-action decision cases, scoped ClawHub ignore policies without re-inclusion overrides, the exact credential-free Claude local MCP configuration, the publishing suite, portable and compatibility manifest consistency, OpenAI public-directory metadata limits, bundled branding assets, and accidental private-path or credential leakage. +The repository validator checks the exact two-skill public discovery surface, keeps contributor-only workflows internal, and checks frontmatter, bundled links, task and trigger fixtures, semantic furniture next-action decision cases, scoped ClawHub ignore policies without re-inclusion overrides, the exact credential-free Claude local MCP configuration, the publishing suite, the exact 46-tool OpenAI annotation and justification packet, portable and compatibility manifest consistency, OpenAI public-directory metadata limits, bundled branding assets, and accidental private-path or credential leakage. diff --git a/skills/VALIDATION.md b/skills/VALIDATION.md index 9e0ebe112..41af09fc5 100644 --- a/skills/VALIDATION.md +++ b/skills/VALIDATION.md @@ -4,9 +4,9 @@ Released package source: **0.1.7**. Latest released package source: **0.1.7**. ` ## OpenAI MCP tool-annotation candidate -The unreleased candidate based on public `main` at `ed562c3a09570a2fe02a6c8c0d56142b224a3b7a` explicitly classifies every registered MCP tool with `readOnlyHint`, `destructiveHint`, and `openWorldHint`. The live `tools/list` regression test enumerated all 46 tools, required an exact classified inventory, and passed. The classifications distinguish 17 closed-world reads, two open-world image-analysis reads, 14 additive closed-world mutations, 12 destructive closed-world operations, and one destructive open-world photo-to-scene operation. +The unreleased submission-preparation candidate based on public `main` at `f107edabb68743290de4ba50b701c2e04869cabb` explicitly classifies every registered MCP tool with `readOnlyHint`, `destructiveHint`, and `openWorldHint`. `plugin-evals/tool-annotation-justifications.json` records the exact three values and a non-empty justification for every hint on all 46 tools. The independent repository policy validator requires the exact inventory, values, keys, ordering, and non-empty justifications; the live `tools/list` regression test requires the packet to match the registered server. The classifications distinguish 17 closed-world reads, two open-world image-analysis reads, 14 additive closed-world mutations, 12 destructive closed-world operations, and one destructive open-world photo-to-scene operation. -`bun test packages/mcp/src/tools/read-tool-annotations.test.ts packages/mcp/src/tools/check-collisions.test.ts` passed 14 tests with 88 assertions. The complete `packages/mcp` suite passed 361 tests with 1,407 assertions, `bun run build` passed, and changed-file Ultracite checks passed. These local checks establish annotation completeness for the tested package source. They do not establish behavior of the production hosted endpoint, an OpenAI Scan Tools result, reviewer access, submission, approval, listing, or release; the publishing suite remains blocked on those prerequisites. +The four focused package-policy files passed 38 tests with 60 assertions. The live annotation regression passed with 324 assertions, and the complete `packages/mcp` suite passed 361 tests with 1,684 assertions. The MCP dependency build, repository check across 2,100 files, package validator, and Claude Code 2.1.267 strict plugin validation also passed. These local checks establish source-level inventory, value, and justification completeness for the tested candidate. They do not establish behavior of the production hosted endpoint, a portal Scan Tools run or approval, domain verification, verified publisher identity, reviewer access, submission, review, publication, listing, or release. The publishing suite records each external prerequisite and remains blocked until they pass. ## Bundle 0.1.7 Claude local MCP release