diff --git a/src/mcp/server.ts b/src/mcp/server.ts index bddf7263de..c5c1646d68 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -114,7 +114,7 @@ import { AGENT_ACTION_CLASSES, isActingAutonomyLevel, resolveAutonomy } from ".. import { MAX_FOCUS_MANIFEST_BYTES } from "../signals/focus-manifest"; import { loadRepoFocusManifest } from "../signals/focus-manifest-loader"; import { buildPredictedGateVerdict } from "../rules/predicted-gate"; -import { buildIssueSlopAssessment, buildSlopAssessment, ISSUE_SLOP_RUBRIC_MARKDOWN, SLOP_RUBRIC_MARKDOWN } from "../signals/slop"; +import { buildIssueSlopAssessment, buildSlopAssessment } from "../signals/slop"; import { buildRepoDataQuality } from "../signals/data-quality"; import { PREFLIGHT_LIMITS } from "../signals/preflight-limits"; import { SCENARIO_MAX_BRANCH_REF_CHARS, SCENARIO_MAX_LINKED_ISSUE_NUMBERS, SCENARIO_MAX_REPO_FULL_NAME_CHARS } from "../scenarios/input-model"; @@ -1100,7 +1100,7 @@ export class GittensoryMcp { "gittensory_check_slop_risk", { description: - "Assess the deterministic slop risk of a planned change from local diff metadata (paths + line counts) + the PR description — an agent-native, source-free quality self-check. Returns slopRisk (0-100), band, findings, and the rubric. No repo data needed.", + "Assess the deterministic slop risk of a planned change from local diff metadata (paths + line counts) + the PR description — an agent-native, source-free quality self-check. Returns band (clean/low/elevated/high) and actionable findings. No repo data needed.", inputSchema: checkSlopRiskShape, outputSchema: checkSlopRiskOutputSchema, }, @@ -1111,7 +1111,7 @@ export class GittensoryMcp { "gittensory_check_issue_slop", { description: - "Assess the deterministic slop risk of an issue from its title + body alone (no repo data) — flags clearly low-effort issues (empty body, an unfilled template) for triage. Returns slopRisk (0-100), band, findings, and the rubric. Advisory-only: issues never block.", + "Assess the deterministic slop risk of an issue from its title + body alone (no repo data) — flags clearly low-effort issues (empty body, an unfilled template) for triage. Returns band and findings. Advisory-only: issues never block.", inputSchema: checkIssueSlopShape, outputSchema: checkIssueSlopOutputSchema, }, @@ -1974,19 +1974,39 @@ export class GittensoryMcp { }; } + // Per-actor rate-limit for slop-check tools: 20 calls per 5 min prevents systematic weight enumeration + // via controlled inputs. Skips gracefully when RATE_LIMITER is unavailable (test / local environments). + private async enforceToolRateLimit(toolName: string): Promise { + if (!this.env.RATE_LIMITER) return; + const key = `mcp-tool:${toolName}:${this.identity.actor}`; + const id = this.env.RATE_LIMITER.idFromName(key); + const response = await this.env.RATE_LIMITER.get(id).fetch("https://rate-limit/check", { + method: "POST", + body: JSON.stringify({ key, limit: 20, windowSeconds: 300 }), + }); + if (response.status === 429) { + const body = (await response.json().catch(() => ({}))) as { retryAfterSeconds?: number }; + throw new Error(`Rate limit exceeded. Retry after ${body.retryAfterSeconds ?? 60}s.`); + } + } + private async checkSlopRisk(input: z.infer>): Promise { + await this.enforceToolRateLimit("gittensory_check_slop_risk"); const assessment = buildSlopAssessment(input); + // Return band + findings only — omit the exact numeric score and rubric thresholds to prevent + // weight reverse-engineering via controlled inputs (#mcp-slop-blunt). return { - summary: `Slop risk: ${assessment.slopRisk}/100 (${assessment.band}).`, - data: { ...assessment, rubric: SLOP_RUBRIC_MARKDOWN } as unknown as Record, + summary: `Slop risk: ${assessment.band}.`, + data: { band: assessment.band, findings: assessment.findings } as unknown as Record, }; } private async checkIssueSlop(input: z.infer>): Promise { + await this.enforceToolRateLimit("gittensory_check_issue_slop"); const assessment = buildIssueSlopAssessment(input); return { - summary: `Issue slop risk: ${assessment.slopRisk}/100 (${assessment.band}).`, - data: { ...assessment, rubric: ISSUE_SLOP_RUBRIC_MARKDOWN } as unknown as Record, + summary: `Issue slop risk: ${assessment.band}.`, + data: { band: assessment.band, findings: assessment.findings } as unknown as Record, }; } diff --git a/test/unit/mcp-check-slop-risk.test.ts b/test/unit/mcp-check-slop-risk.test.ts index 4dcb97c89d..2e7373b17a 100644 --- a/test/unit/mcp-check-slop-risk.test.ts +++ b/test/unit/mcp-check-slop-risk.test.ts @@ -14,19 +14,20 @@ async function connect() { } describe("MCP gittensory_check_slop_risk", () => { - it("assesses slop from local diff metadata (no repo/auth needed) and returns the rubric", async () => { + it("assesses slop from local diff metadata (no repo/auth needed) and returns band + findings", async () => { const client = await connect(); const result = await client.callTool({ name: "gittensory_check_slop_risk", arguments: { changedFiles: [{ path: "src/api/routes.ts", additions: 6, deletions: 1 }], description: "" }, }); expect(result.isError).toBeFalsy(); - const data = result.structuredContent as { slopRisk: number; band: string; findings: Array<{ code: string }>; rubric: string }; - expect(data.slopRisk).toBeGreaterThan(0); + const data = result.structuredContent as { band: string; findings: Array<{ code: string }> }; expect(["low", "elevated", "high"]).toContain(data.band); // Code change + no tests + empty description → both signals. expect(data.findings.map((f) => f.code)).toEqual(expect.arrayContaining(["missing_test_evidence", "empty_pr_description"])); - expect(data.rubric).toContain("slop assessment rubric"); + // Blunted: exact numeric score and rubric thresholds are NOT returned (#mcp-slop-blunt). + expect(data).not.toHaveProperty("slopRisk"); + expect(data).not.toHaveProperty("rubric"); expect(JSON.stringify(data)).not.toMatch(/wallet|hotkey|reward|payout|trust score/i); }); @@ -39,21 +40,23 @@ describe("MCP gittensory_check_slop_risk", () => { description: "Adds a retry path with regression coverage.", }, }); - const data = result.structuredContent as { slopRisk: number; band: string }; - expect(data.slopRisk).toBe(0); + const data = result.structuredContent as { band: string }; expect(data.band).toBe("clean"); + expect(data).not.toHaveProperty("slopRisk"); }); }); describe("MCP gittensory_check_issue_slop (#533)", () => { - it("flags a low-effort issue (empty body) from title+body alone and returns the issue rubric", async () => { + it("flags a low-effort issue (empty body) from title+body alone and returns band + findings", async () => { const client = await connect(); const result = await client.callTool({ name: "gittensory_check_issue_slop", arguments: { title: "broken", body: " " } }); expect(result.isError).toBeFalsy(); - const data = result.structuredContent as { slopRisk: number; band: string; findings: Array<{ code: string }>; rubric: string }; - expect(data.slopRisk).toBeGreaterThan(0); + const data = result.structuredContent as { band: string; findings: Array<{ code: string }> }; + expect(["low", "elevated", "high"]).toContain(data.band); expect(data.findings.map((f) => f.code)).toEqual(["empty_issue_body"]); - expect(data.rubric).toContain("issue slop triage rubric"); + // Blunted: exact numeric score and rubric thresholds are NOT returned (#mcp-slop-blunt). + expect(data).not.toHaveProperty("slopRisk"); + expect(data).not.toHaveProperty("rubric"); expect(JSON.stringify(data)).not.toMatch(/wallet|hotkey|reward|payout|trust score/i); }); @@ -63,8 +66,8 @@ describe("MCP gittensory_check_issue_slop (#533)", () => { name: "gittensory_check_issue_slop", arguments: { title: "500 on save", body: "Clicking Save on /settings returns a 500; expected a redirect. Repro: open /settings, submit." }, }); - const data = result.structuredContent as { slopRisk: number; band: string }; - expect(data.slopRisk).toBe(0); + const data = result.structuredContent as { band: string }; expect(data.band).toBe("clean"); + expect(data).not.toHaveProperty("slopRisk"); }); }); diff --git a/test/unit/mcp-output-schemas.test.ts b/test/unit/mcp-output-schemas.test.ts index 02eb3c9626..73ad8134c2 100644 --- a/test/unit/mcp-output-schemas.test.ts +++ b/test/unit/mcp-output-schemas.test.ts @@ -535,3 +535,71 @@ describe("MCP output schemas validate on real tool calls (#550)", () => { expect(fetched.structuredContent).toBeDefined(); }, 30_000); }); + +// ── Slop oracle blunting (#mcp-slop-blunt) ──────────────────────────────────── + +function mockRateLimiter(status: number, body: Record = {}): NonNullable { + return { + idFromName: (name: string) => name as unknown as DurableObjectId, + get: () => ({ + async fetch(_url: string, _init?: RequestInit) { + return Response.json(body, { status }); + }, + }), + } as unknown as NonNullable; +} + +async function callSlopTool(env: Env, toolName: string, args: Record) { + const server = new GittensoryMcp(env).createServer(); + const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair(); + await server.connect(serverTransport); + const client = new Client({ name: "test", version: "0.0.1" }, { capabilities: {} }); + await client.connect(clientTransport); + return client.callTool({ name: toolName, arguments: args }); +} + +describe("MCP slop oracle blunting", () => { + const slopArgs = { changedFiles: [{ path: "src/foo.ts", additions: 5 }] }; + + it("omits the exact slopRisk score and rubric from gittensory_check_slop_risk response", async () => { + const result = await callSlopTool(createTestEnv(), "gittensory_check_slop_risk", slopArgs); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as Record; + expect(data).not.toHaveProperty("slopRisk"); + expect(data).not.toHaveProperty("rubric"); + expect(data).toHaveProperty("band"); + expect(data).toHaveProperty("findings"); + const text = (result.content as Array<{ type: string; text: string }>)[0]?.text ?? ""; + expect(text).not.toMatch(/\/100/); + }); + + it("omits the exact slopRisk score and rubric from gittensory_check_issue_slop response", async () => { + const result = await callSlopTool(createTestEnv(), "gittensory_check_issue_slop", { title: "Fix bug", body: "Description." }); + expect(result.isError).toBeFalsy(); + const data = result.structuredContent as Record; + expect(data).not.toHaveProperty("slopRisk"); + expect(data).not.toHaveProperty("rubric"); + expect(data).toHaveProperty("band"); + }); + + it("skips the tool rate-limit when RATE_LIMITER is absent (test/local env)", async () => { + // createTestEnv() has no RATE_LIMITER — enforceToolRateLimit must return early without throwing. + const result = await callSlopTool(createTestEnv(), "gittensory_check_slop_risk", slopArgs); + expect(result.isError).toBeFalsy(); + }); + + it("allows the call when the tool rate-limit returns 200", async () => { + const env = createTestEnv({ RATE_LIMITER: mockRateLimiter(200, { allowed: true, remaining: 19 }) }); + const result = await callSlopTool(env, "gittensory_check_slop_risk", slopArgs); + expect(result.isError).toBeFalsy(); + expect((result.structuredContent as Record).band).toBeDefined(); + }); + + it("returns an error when the tool rate-limit returns 429", async () => { + const env = createTestEnv({ RATE_LIMITER: mockRateLimiter(429, { retryAfterSeconds: 42 }) }); + const result = await callSlopTool(env, "gittensory_check_slop_risk", slopArgs); + expect(result.isError).toBe(true); + const text = (result.content as Array<{ type: string; text: string }>)[0]?.text ?? ""; + expect(text).toMatch(/rate limit exceeded/i); + }); +});