diff --git a/apps/gittensory-ui/src/lib/command-reference.ts b/apps/gittensory-ui/src/lib/command-reference.ts index 9659c69c95..38b6515f05 100644 --- a/apps/gittensory-ui/src/lib/command-reference.ts +++ b/apps/gittensory-ui/src/lib/command-reference.ts @@ -150,7 +150,13 @@ export const ACTION_COMMAND_ENTRIES = [ description: "Explain a specific review finding; supply the finding reference in trailing text.", }, + { + id: "generate-tests", + title: "Generate E2E tests", + description: + "Generate an AI E2E test for this PR's changed behavior and post it as a reply comment (maintainer-only).", + }, ] as const; export const ACTION_COMMAND_LIST = - "@gittensory gate-override\n@gittensory review\n@gittensory pause\n@gittensory resume\n@gittensory resolve\n@gittensory configuration\n@gittensory explain"; + "@gittensory gate-override\n@gittensory review\n@gittensory pause\n@gittensory resume\n@gittensory resolve\n@gittensory configuration\n@gittensory explain\n@gittensory generate-tests"; diff --git a/packages/gittensory-engine/src/settings/command-authorization.ts b/packages/gittensory-engine/src/settings/command-authorization.ts index b3b4c861a7..7f4c0de1a0 100644 --- a/packages/gittensory-engine/src/settings/command-authorization.ts +++ b/packages/gittensory-engine/src/settings/command-authorization.ts @@ -24,6 +24,14 @@ export const DEFAULT_COMMAND_AUTHORIZATION_POLICY: RepositoryCommandAuthorizatio resolve: ["maintainer", "collaborator"], configuration: ["maintainer", "collaborator"], explain: ["maintainer", "collaborator"], + // #4195 (part of the #4189 E2E-test-generation epic): deliberately NARROWER than every command above -- + // "maintainer" ONLY, excluding "collaborator" and "confirmed_miner". This command can write real content + // (a generated test) attributed to the PR; a repo could grant a contributor/miner collaborator-level + // push access, and that tier must not be able to invoke test generation for their own scored PR (the + // exact loophole a click-to-generate button would otherwise open). The existing + // `maintainer_command_requires_maintainer` guard below already denies the PR's own author when they + // don't independently hold the `maintainer` role, so no bespoke pr_author check is needed here. + "generate-tests": ["maintainer"], }, }; diff --git a/src/github/commands.ts b/src/github/commands.ts index 5888db0d22..10e2f95206 100644 --- a/src/github/commands.ts +++ b/src/github/commands.ts @@ -98,6 +98,11 @@ export const GITTENSORY_ACTION_COMMAND_CATALOG = [ title: "Explain finding", description: "Explain a specific review finding; supply the finding reference in trailing text.", }, + { + id: "generate-tests", + title: "Generate E2E tests", + description: "Generate an AI E2E test for this PR's changed behavior and post it as a reply comment (maintainer-only).", + }, ] as const; export type GittensoryActionCommandName = (typeof GITTENSORY_ACTION_COMMAND_CATALOG)[number]["id"]; diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 1e66396fb1..06a7c19a4b 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -488,6 +488,8 @@ import { computeImpactMap, type ImpactMapEntry } from "../review/impact-map"; import { formatImpactMapPromptSection, shouldComputeImpactMap } from "../review/impact-map-wire"; import { shouldEmitFixHandoff } from "../review/fix-handoff"; import { buildFixHandoffBlocks } from "../review/fix-handoff-render"; +import { buildE2eTestGenCommentBody } from "../review/e2e-test-gen-render"; +import { resolveE2eTestGenInstructions, runGittensoryE2eTestGeneration } from "../services/ai-e2e-test-gen"; import { buildRepoCultureProfileContext, isRepoCultureProfileEnabled, @@ -5669,6 +5671,7 @@ async function processGitHubWebhook( if (eventName === "issue_comment" && (await maybeProcessResolveCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, eventName, action: payload.action, installationId: payload.installation?.id, repositoryFullName: payload.repository?.full_name, payloadHash: "processed", status: "processed" }); return; } if (eventName === "issue_comment" && (await maybeProcessExplainCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, eventName, action: payload.action, installationId: payload.installation?.id, repositoryFullName: payload.repository?.full_name, payloadHash: "processed", status: "processed" }); return; } + if (eventName === "issue_comment" && (await maybeProcessGenerateTestsCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, eventName, action: payload.action, installationId: payload.installation?.id, repositoryFullName: payload.repository?.full_name, payloadHash: "processed", status: "processed" }); return; } if (eventName === "issue_comment" && (await maybeProcessReviewCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, eventName, action: payload.action, installationId: payload.installation?.id, repositoryFullName: payload.repository?.full_name, payloadHash: "processed", status: "processed" }); return; } if (eventName === "issue_comment" && (await maybeProcessPauseCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, eventName, action: payload.action, installationId: payload.installation?.id, repositoryFullName: payload.repository?.full_name, payloadHash: "processed", status: "processed" }); return; } if (eventName === "issue_comment" && (await maybeProcessResumeCommand(env, deliveryId, payload))) { await recordWebhookEvent(env, { deliveryId, eventName, action: payload.action, installationId: payload.installation?.id, repositoryFullName: payload.repository?.full_name, payloadHash: "processed", status: "processed" }); return; } @@ -11406,6 +11409,113 @@ async function recordFindingExplainedSkip(env: Env, deliveryId: string, repoFull await recordGithubProductUsage(env, "finding_explained_skipped", { actor, repoFullName, targetKey, outcome: "skipped", metadata: { reason } }); } +/** + * `@gittensory generate-tests` (#4195, part of the #4189 epic): on-demand, MAINTAINER-ONLY AI-generated E2E + * test coverage for this PR's changed behavior, posted as its own reply comment — mirroring + * `maybeProcessExplainCommand`'s classify → authorize → act → audit shape exactly, but posting fresh + * generated content rather than explaining already-published findings. + * + * Deliberately does NOT splice into the automated review's sticky unified comment (unlike fix-handoff): + * this is an explicit, cost-bearing, maintainer-triggered action, not something derived for free from data + * the regular review pass already computed — see `explain`/`configuration` for the same "own dedicated + * reply comment" precedent for on-demand actions. + */ +async function maybeProcessGenerateTestsCommand(env: Env, deliveryId: string, payload: GitHubWebhookPayload): Promise { + const command = parseGittensoryMentionCommand(payload.comment?.body); + if (!command || command.name !== "generate-tests") return false; + const { classifyPrCommandRequest } = await import("../github/pr-command-request"); + const req = classifyPrCommandRequest(payload, getInstallationId(payload)); + if (!req.ok) { + await recordGenerateTestsSkip(env, deliveryId, req.repoFullName, req.targetKey, req.actor, req.reason); + return true; + } + const targetKey = `${req.repoFullName}#${req.pr.number}`; + const [pr, settings] = await Promise.all([getPullRequest(env, req.repoFullName, req.pr.number), resolveRepositorySettings(env, req.repoFullName)]); + if (!pr) { + await recordGenerateTestsSkip(env, deliveryId, req.repoFullName, targetKey, req.actor, "cached_pr_missing"); + return true; + } + const { authorization } = await authorizePrActionActor({ env, deliveryId, installationId: req.installationId, repoFullName: req.repoFullName, issue: payload.issue!, actor: req.actor, commandName: "generate-tests" as GittensoryMentionCommandName, settings, pr }); + if (!authorization.authorized) { + await recordAuditEvent(env, { eventType: "github_app.e2e_tests_generation_denied", actor: req.actor, targetKey, outcome: "denied", detail: authorization.reason, metadata: { deliveryId, repoFullName: req.repoFullName, allowedRoles: commandAuthorizationAllowedRoles(settings.commandAuthorization, "generate-tests") } }); + await recordGithubProductUsage(env, "e2e_tests_generation_denied", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "denied", metadata: { reason: authorization.reason, actorKind: authorization.actorKind } }); + return true; + } + const manifest = await loadRepoFocusManifest(env, req.repoFullName).catch(() => null); + if (!resolveConvergedFeature(env, manifest, "e2eTests", req.repoFullName)) { + await postGenerateTestsNotEnabledComment(env, req.installationId, req.repoFullName, req.pr.number); + await recordGenerateTestsSkip(env, deliveryId, req.repoFullName, targetKey, req.actor, "feature_disabled"); + return true; + } + const files = await listPullRequestFiles(env, req.repoFullName, req.pr.number); + const changedPaths = files.map((file) => file.path); + // BYOK resolution mirrors runAiReviewForAdvisory's own (re-resolved per-caller is this codebase's + // established convention for this exact 3-line block — see e.g. the vision-capture caller above). + const storedKey = settings.aiReviewByok ? await getDecryptedRepositoryAiKey(env, req.repoFullName) : null; + const providerKey = + storedKey && (!settings.aiReviewProvider || settings.aiReviewProvider === storedKey.provider) + ? { provider: storedKey.provider, key: storedKey.key, model: settings.aiReviewModel ?? storedKey.model } + : null; + const result = await runGittensoryE2eTestGeneration(env, { + repoFullName: req.repoFullName, + prNumber: req.pr.number, + title: pr.title, + body: pr.body, + files: files.map((file) => ({ path: file.path, patch: typeof file.payload?.patch === "string" ? file.payload.patch : undefined })), + instructions: resolveE2eTestGenInstructions(manifest?.review, changedPaths), + actor: req.actor, + providerKey, + }); + const testSource = result.status === "ok" ? result.testSource : null; + const body = buildE2eTestGenCommentBody({ actor: req.actor, testSource }); + try { + await createIssueComment(env, req.installationId, req.repoFullName, req.pr.number, sanitizePublicComment(body)); + } catch (error) { + // sanitizePublicComment THROWS on a forbidden term rather than stripping it -- generated test source is + // far less predictable than this codebase's other curated comment content, so failing closed to a safe + // withheld-content note (never the raw error, never the raw generated text) is the right degrade here. + await createIssueComment( + env, + req.installationId, + req.repoFullName, + req.pr.number, + sanitizePublicComment(buildE2eTestGenCommentBody({ actor: req.actor, testSource: null })), + ); + console.log(JSON.stringify({ event: "e2e_test_gen_comment_withheld", repoFullName: req.repoFullName, pr: req.pr.number, error: errorMessage(error) })); + } + await recordAuditEvent(env, { + eventType: "github_app.e2e_tests_generation", + actor: req.actor, + targetKey, + outcome: "completed", + detail: testSource ? "Generated an E2E test." : `No usable test generated (${result.status}).`, + metadata: { deliveryId, repoFullName: req.repoFullName, status: result.status, byok: Boolean(providerKey) }, + }); + await recordGithubProductUsage(env, "e2e_tests_generation", { actor: req.actor, repoFullName: req.repoFullName, targetKey, outcome: "completed", metadata: { status: result.status, generated: Boolean(testSource) } }); + return true; +} + +async function postGenerateTestsNotEnabledComment(env: Env, installationId: number, repoFullName: string, prNumber: number): Promise { + const body = sanitizePublicComment( + [ + AGENT_COMMAND_COMMENT_MARKER, + "", + "> [!NOTE]", + "> **E2E test generation is not enabled for this repository**", + "> Ask a maintainer to enable `features.e2eTests` in `.gittensory.yml` (the operator's global flag must also be on).", + "", + "---", + gittensoryFooter(), + ].join("\n"), + ); + await createIssueComment(env, installationId, repoFullName, prNumber, body); +} + +async function recordGenerateTestsSkip(env: Env, deliveryId: string, repoFullName: string | null, targetKey: string | null, actor: string | null, reason: string): Promise { + await recordAuditEvent(env, { eventType: "github_app.e2e_tests_generation_skipped", actor, targetKey, outcome: "completed", detail: reason, metadata: { deliveryId, repoFullName, reason } }); + await recordGithubProductUsage(env, "e2e_tests_generation_skipped", { actor, repoFullName, targetKey, outcome: "skipped", metadata: { reason } }); +} + async function appendPublishedAiReviewFindingsForResolve( env: Env, repoFullName: string, diff --git a/src/review/e2e-test-gen-render.ts b/src/review/e2e-test-gen-render.ts new file mode 100644 index 0000000000..d6f51326de --- /dev/null +++ b/src/review/e2e-test-gen-render.ts @@ -0,0 +1,55 @@ +// Public-safe rendering for AI-generated E2E test coverage (#4193, part of the #4189 epic). +// +// Unlike fix-handoff (which splices a block into the automated review's sticky unified comment), this +// renders its OWN dedicated reply comment for the `@gittensory generate-tests` command (#4195) — a +// maintainer-triggered, on-demand action, not something that runs on every automated review pass. This +// mirrors how `explain`/`configuration` already post their own on-demand response comments rather than +// editing the main review comment (see `maybeProcessExplainCommand` in `src/queue/processors.ts`). +// +// This layer never re-derives safety: it trusts that #4191's `parseE2eTestGenResponse` already validated +// the test source is plausible Playwright before this ever sees it, and that #4195's caller already +// resolved authorization — this file only turns already-decided content into a public-safe comment body. +import { AGENT_COMMAND_COMMENT_MARKER } from "../github/comments"; +import { gittensoryFooter } from "../github/footer"; + +export type E2eTestGenCommentInput = { + actor: string; + /** The generated test source, or null when generation ran but produced nothing usable. */ + testSource: string | null; + framework?: string | undefined; +}; + +/** + * Build the PR-comment body for a `@gittensory generate-tests` result. A null `testSource` renders a + * clear "nothing usable" note rather than silently posting no comment at all — the maintainer who invoked + * the command should always get a response, even a negative one. + */ +export function buildE2eTestGenCommentBody(input: E2eTestGenCommentInput): string { + const framework = input.framework?.trim() || "Playwright"; + if (!input.testSource) { + return [ + AGENT_COMMAND_COMMENT_MARKER, + "", + "> [!NOTE]", + `> **E2E test generation for @${input.actor} did not produce a usable result**`, + `> The model's output didn't parse as valid ${framework} source — try again, or add the test by hand.`, + "", + "---", + gittensoryFooter(), + ].join("\n"); + } + return [ + AGENT_COMMAND_COMMENT_MARKER, + "", + "> [!NOTE]", + `> **AI-generated ${framework} test for @${input.actor}**`, + "> This is a suggestion, not a guarantee — review it like any other test before merging.", + "", + "```typescript", + input.testSource, + "```", + "", + "---", + gittensoryFooter(), + ].join("\n"); +} diff --git a/src/settings/command-authorization.ts b/src/settings/command-authorization.ts index e3b8931386..84d935c32f 100644 --- a/src/settings/command-authorization.ts +++ b/src/settings/command-authorization.ts @@ -24,6 +24,14 @@ export const DEFAULT_COMMAND_AUTHORIZATION_POLICY: RepositoryCommandAuthorizatio resolve: ["maintainer", "collaborator"], configuration: ["maintainer", "collaborator"], explain: ["maintainer", "collaborator"], + // #4195 (part of the #4189 E2E-test-generation epic): deliberately NARROWER than every command above -- + // "maintainer" ONLY, excluding "collaborator" and "confirmed_miner". This command can write real content + // (a generated test) attributed to the PR; a repo could grant a contributor/miner collaborator-level + // push access, and that tier must not be able to invoke test generation for their own scored PR (the + // exact loophole a click-to-generate button would otherwise open). The existing + // `maintainer_command_requires_maintainer` guard below already denies the PR's own author when they + // don't independently hold the `maintainer` role, so no bespoke pr_author check is needed here. + "generate-tests": ["maintainer"], }, }; diff --git a/test/unit/e2e-test-gen-render.test.ts b/test/unit/e2e-test-gen-render.test.ts new file mode 100644 index 0000000000..2fc6864789 --- /dev/null +++ b/test/unit/e2e-test-gen-render.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vitest"; +import { buildE2eTestGenCommentBody } from "../../src/review/e2e-test-gen-render"; +import { PR_PANEL_COMMENT_MARKER } from "../../src/github/comments"; + +describe("buildE2eTestGenCommentBody", () => { + it("renders the generated test source in a fenced code block, defaulting the framework to Playwright", () => { + const body = buildE2eTestGenCommentBody({ actor: "maintainer", testSource: "test('x', () => {});" }); + expect(body).toContain(PR_PANEL_COMMENT_MARKER); + expect(body).toContain("AI-generated Playwright test for @maintainer"); + expect(body).toContain("```typescript\ntest('x', () => {});\n```"); + }); + + it("uses a custom framework name when provided", () => { + const body = buildE2eTestGenCommentBody({ actor: "maintainer", testSource: "it('x', () => {});", framework: "Cypress" }); + expect(body).toContain("AI-generated Cypress test for @maintainer"); + }); + + it("renders a not-usable note (no code fence) when testSource is null", () => { + const body = buildE2eTestGenCommentBody({ actor: "maintainer", testSource: null }); + expect(body).toContain(PR_PANEL_COMMENT_MARKER); + expect(body).toContain("did not produce a usable result"); + expect(body).not.toContain("```"); + }); + + it("names the configured framework in the not-usable note too", () => { + const body = buildE2eTestGenCommentBody({ actor: "maintainer", testSource: null, framework: "Cypress" }); + expect(body).toContain("didn't parse as valid Cypress source"); + }); +}); diff --git a/test/unit/gen-command-reference-script.test.ts b/test/unit/gen-command-reference-script.test.ts index 39cedeccaf..619361a4e1 100644 --- a/test/unit/gen-command-reference-script.test.ts +++ b/test/unit/gen-command-reference-script.test.ts @@ -94,12 +94,12 @@ describe("gen-command-reference script (#3046)", () => { expect(actionCommands).toHaveLength(7); }); - it("extracts the real 10 public + 9 maintainer-only + 7 action commands from the real repo source", () => { + it("extracts the real 10 public + 9 maintainer-only + 8 action commands from the real repo source", () => { const { publicCommands, maintainerCommands, actionCommands } = collectCommandCatalogs({ rootDir: process.cwd() }); expect(publicCommands).toHaveLength(10); expect(maintainerCommands).toHaveLength(9); - expect(actionCommands).toHaveLength(7); + expect(actionCommands).toHaveLength(8); expect(publicCommands.map((c: CommandCatalogEntry) => c.id)).toEqual([ "help", "ask", diff --git a/test/unit/queue.test.ts b/test/unit/queue.test.ts index e78d446cf9..25ec17daac 100644 --- a/test/unit/queue.test.ts +++ b/test/unit/queue.test.ts @@ -47,6 +47,7 @@ import { upsertPullRequestFile, upsertPullRequestFromGitHub, upsertIssueWatchSubscription, + upsertRepositoryAiKey, upsertRepositorySettings, upsertRepositoryFromGitHub, putCachedAiReview, @@ -24757,6 +24758,338 @@ describe("queue processors", () => { }); }); + // #4195 (part of the #4189 E2E-test-generation epic): `@gittensory generate-tests` -- on-demand, + // MAINTAINER-ONLY AI-generated E2E test coverage, posted as its own reply comment. Mirrors the explain + // harness above (classify -> authorize -> act -> audit), but with the authorization tier deliberately + // narrowed to ["maintainer"] only -- no collaborator, no confirmed_miner -- and a real (mocked) model call. + describe("@gittensory generate-tests (#4195)", () => { + async function seedGenerateTestsPr(env: Env, repoFullName: string, prNumber: number, headSha: string, authorLogin = "contributor") { + const slash = repoFullName.indexOf("/"); + const owner = repoFullName.slice(0, slash); + const name = repoFullName.slice(slash + 1); + await upsertRepositoryFromGitHub(env, { name, full_name: repoFullName, private: false, owner: { login: owner } }, 123); + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "advisory", aiReviewMode: "advisory" }); + await upsertPullRequestFromGitHub(env, repoFullName, { number: prNumber, title: "Add retry to checkout", state: "open", user: { login: authorLogin }, author_association: "CONTRIBUTOR", head: { sha: headSha }, labels: [], body: "Retries the payment call once on a 5xx." }); + await upsertPullRequestFile(env, { repoFullName, pullNumber: prNumber, path: "src/checkout.ts", status: "modified", additions: 3, deletions: 0, changes: 3, payload: { patch: "+function retryPayment() {\n+ return true;\n+}" } }); + // A renamed-with-no-patch file (GitHub omits `patch` for pure renames) -- exercises the + // payload?.patch-is-not-a-string branch in the files.map() that builds E2eTestGenChangedFile[]. + await upsertPullRequestFile(env, { repoFullName, pullNumber: prNumber, path: "src/renamed.ts", status: "renamed", additions: 0, deletions: 0, changes: 0, payload: {} }); + await upsertRepoFocusManifest(env, repoFullName, { features: { e2eTests: true } }); + } + const generateTestsWebhook = (repoFullName: string, prNumber: number, actor: string, opts: { association?: string; bot?: boolean; commenterIsAuthor?: boolean } = {}) => ({ + type: "github-webhook" as const, + deliveryId: `generate-tests-${prNumber}-${actor}`, + eventName: "issue_comment" as const, + payload: { + action: "created", + installation: { id: 123, account: { login: repoFullName.slice(0, repoFullName.indexOf("/")), id: 1, type: "User" } }, + repository: { name: repoFullName.slice(repoFullName.indexOf("/") + 1), full_name: repoFullName, private: false, owner: { login: repoFullName.slice(0, repoFullName.indexOf("/")) } }, + issue: { number: prNumber, title: "Add retry to checkout", state: "open", user: { login: opts.commenterIsAuthor ? actor : "contributor" }, pull_request: {} }, + comment: { id: prNumber * 10, body: "@gittensory generate-tests", author_association: opts.association ?? "NONE", user: { login: actor, type: opts.bot ? "Bot" : "User" } }, + sender: { login: actor, type: opts.bot ? "Bot" : "User" }, + }, + }) as unknown as Parameters[1]; + const VALID_TEST_SOURCE = "import { test, expect } from '@playwright/test';\n\ntest('checkout retries on failure', async ({ page }) => {\n await page.goto('/checkout');\n await expect(page.getByRole('button', { name: 'Pay' })).toBeVisible();\n});"; + + it("generates and posts an E2E test for an authorized maintainer, and records a completed audit event", async () => { + const repoFullName = "JSONbored/gen-tests-4195-ok"; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "```typescript\n" + VALID_TEST_SOURCE + "\n```" }) } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedGenerateTestsPr(env, repoFullName, 4195, "gen-tests-4195-ok"); + let postedBody = ""; + let posted = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4195/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4195/comments") && method === "POST") { posted += 1; postedBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 41950 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4195, "maintainer", { association: "MEMBER" })); + + expect(posted).toBe(1); + expect(postedBody).toContain("AI-generated Playwright test for @maintainer"); + expect(postedBody).toContain("test('checkout retries on failure'"); + const audited = await env.DB.prepare("select outcome, metadata_json from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ outcome: string; metadata_json: string }>(); + expect(audited?.outcome).toBe("completed"); + expect(JSON.parse(audited?.metadata_json ?? "{}")).toMatchObject({ status: "ok", byok: false }); + }); + + it("denies a collaborator-tier actor (write permission, not the PR author) — narrower than every other command", async () => { + const repoFullName = "JSONbored/gen-tests-4195-collab"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedGenerateTestsPr(env, repoFullName, 4196, "gen-tests-4195-collab"); + let posted = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/writer/permission")) return Response.json({ permission: "write" }); + if (url.includes("/issues/4196/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4196/comments") && method === "POST") { posted += 1; return Response.json({ id: 41960 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4196, "writer", { association: "COLLABORATOR" })); + + expect(posted).toBe(0); // denied before any generation or comment + const denied = await env.DB.prepare("select outcome, detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_denied").first<{ outcome: string; detail: string }>(); + expect(denied?.outcome).toBe("denied"); + }); + + it("denies the PR's own author even though they authored it — the exact loophole a click-to-generate button must not open", async () => { + const repoFullName = "JSONbored/gen-tests-4195-author"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedGenerateTestsPr(env, repoFullName, 4197, "gen-tests-4195-author", "contributor"); + let posted = 0; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + // No collaborator/permission relationship at all -- a plain contributor commenting on their own PR. + if (url.includes("/collaborators/contributor/permission")) return new Response("not found", { status: 404 }); + if (url.includes("/issues/4197/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4197/comments") && method === "POST") { posted += 1; return Response.json({ id: 41970 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4197, "contributor", { association: "NONE", commenterIsAuthor: true })); + + expect(posted).toBe(0); + const denied = await env.DB.prepare("select detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_denied").first<{ detail: string }>(); + expect(denied?.detail).toBe("maintainer_command_requires_maintainer"); + }); + + it("posts a not-enabled note (no generation call) when features.e2eTests is off for the repo", async () => { + const repoFullName = "JSONbored/gen-tests-4195-disabled"; + const run = vi.fn(); + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), AI: { run } as unknown as Ai, GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + const slash = repoFullName.indexOf("/"); + await upsertRepositoryFromGitHub(env, { name: repoFullName.slice(slash + 1), full_name: repoFullName, private: false, owner: { login: repoFullName.slice(0, slash) } }, 123); + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "advisory", aiReviewMode: "advisory" }); + await upsertPullRequestFromGitHub(env, repoFullName, { number: 4198, title: "x", state: "open", user: { login: "contributor" }, author_association: "CONTRIBUTOR", head: { sha: "gen-tests-4195-disabled" }, labels: [], body: "x" }); + // Deliberately no upsertRepoFocusManifest features.e2eTests override -- stays off (no allowlist either). + let postedBody = ""; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4198/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4198/comments") && method === "POST") { postedBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 41980 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4198, "maintainer", { association: "MEMBER" })); + + expect(postedBody).toContain("E2E test generation is not enabled for this repository"); + expect(run).not.toHaveBeenCalled(); + const skipped = await env.DB.prepare("select detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_skipped").first<{ detail: string }>(); + expect(skipped?.detail).toBe("feature_disabled"); + }); + + it("posts a did-not-produce-a-usable-result note when the model output never parses", async () => { + const repoFullName = "JSONbored/gen-tests-4195-garbage"; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "not a test file" }) } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + await seedGenerateTestsPr(env, repoFullName, 4199, "gen-tests-4195-garbage"); + let postedBody = ""; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4199/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4199/comments") && method === "POST") { postedBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 41990 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4199, "maintainer", { association: "MEMBER" })); + + expect(postedBody).toContain("did not produce a usable result"); + const audited = await env.DB.prepare("select metadata_json from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ metadata_json: string }>(); + expect(JSON.parse(audited?.metadata_json ?? "{}")).toMatchObject({ status: "ok" }); + }); + + it("skips cleanly when the cached PR record is missing", async () => { + const repoFullName = "JSONbored/gen-tests-4195-nopr"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + const slash = repoFullName.indexOf("/"); + await upsertRepositoryFromGitHub(env, { name: repoFullName.slice(slash + 1), full_name: repoFullName, private: false, owner: { login: repoFullName.slice(0, slash) } }, 123); + // No upsertPullRequestFromGitHub -- the PR was never cached. + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4200, "maintainer", { association: "MEMBER" })); + + const skipped = await env.DB.prepare("select detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_skipped").first<{ detail: string }>(); + expect(skipped?.detail).toBe("cached_pr_missing"); + }); + + it("declines (returns false) for a non-command comment, claiming nothing", async () => { + const repoFullName = "JSONbored/gen-tests-4195-decline"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedGenerateTestsPr(env, repoFullName, 4201, "gen-tests-4195-decline"); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4201/comments")) return Response.json([]); + return new Response("not found", { status: 404 }); + }); + const webhook = generateTestsWebhook(repoFullName, 4201, "maintainer", { association: "MEMBER" }); + (webhook as unknown as { payload: { comment: { body: string } } }).payload.comment.body = "just chatting, no mention here"; + + await processJob(env, webhook); + + const rows = await env.DB.prepare("select count(*) as n from audit_events where event_type like 'github_app.e2e_tests_generation%'").first<{ n: number }>(); + expect(rows?.n).toBe(0); + }); + + it("skips cleanly when the comment classifies as invalid (a bot posted the mention)", async () => { + const repoFullName = "JSONbored/gen-tests-4195-bot"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedGenerateTestsPr(env, repoFullName, 4202, "gen-tests-4195-bot"); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4202, "some-bot[bot]", { association: "NONE", bot: true })); + + const skipped = await env.DB.prepare("select detail from audit_events where event_type = ?").bind("github_app.e2e_tests_generation_skipped").first<{ detail: string }>(); + expect(skipped?.detail).toBe("bot_author"); + }); + + it("uses the maintainer's BYOK frontier model (not Workers AI) when aiReviewByok is on and a key is configured", async () => { + const repoFullName = "JSONbored/gen-tests-4195-byok"; + const run = vi.fn(); // Workers AI must NOT be used when BYOK is configured + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + TOKEN_ENCRYPTION_SECRET: "gen-tests-byok-test-encryption-secret-32b", + }); + await seedGenerateTestsPr(env, repoFullName, 4203, "gen-tests-4195-byok"); + // aiReviewProvider set AND matching the stored key's provider -- exercises the "explicit provider + // pin agrees with the stored key" arm, distinct from the (also-tested-elsewhere) "no pin configured" + // default arm. + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "advisory", aiReviewMode: "advisory", aiReviewByok: true, aiReviewProvider: "anthropic" }); + await upsertRepositoryAiKey(env, { repoFullName, provider: "anthropic", key: "sk-ant-byok-gen-tests-9999", model: null }); + let postedBody = ""; + const fetchMock = vi.fn(async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("api.anthropic.com")) return Response.json({ content: [{ type: "text", text: "```typescript\n" + VALID_TEST_SOURCE + "\n```" }] }); + if (url.includes("/issues/4203/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4203/comments") && method === "POST") { postedBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 42030 }); } + return new Response("not found", { status: 404 }); + }); + vi.stubGlobal("fetch", fetchMock); + + await processJob(env, generateTestsWebhook(repoFullName, 4203, "maintainer", { association: "MEMBER" })); + + expect(run).not.toHaveBeenCalled(); + expect(postedBody).toContain("test('checkout retries on failure'"); + const audited = await env.DB.prepare("select metadata_json from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ metadata_json: string }>(); + expect(JSON.parse(audited?.metadata_json ?? "{}")).toMatchObject({ byok: true }); + }); + + it("degrades to the not-usable-result note when the feature is on but no AI provider is configured at all", async () => { + const repoFullName = "JSONbored/gen-tests-4195-unavailable"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedGenerateTestsPr(env, repoFullName, 4204, "gen-tests-4195-unavailable"); // no env.AI, no BYOK key + let postedBody = ""; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4204/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4204/comments") && method === "POST") { postedBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 42040 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4204, "maintainer", { association: "MEMBER" })); + + expect(postedBody).toContain("did not produce a usable result"); + const audited = await env.DB.prepare("select metadata_json from audit_events where event_type = ?").bind("github_app.e2e_tests_generation").first<{ metadata_json: string }>(); + expect(JSON.parse(audited?.metadata_json ?? "{}")).toMatchObject({ status: "unavailable" }); + }); + + it("generates via the GITTENSORY_REVIEW_REPOS allowlist default when no manifest is published at all", async () => { + // No upsertRepoFocusManifest call -- loadRepoFocusManifest resolves null, so manifest?.review (fed to + // resolveE2eTestGenInstructions) and the e2eTests feature gate itself both take their null/allowlist path. + const repoFullName = "JSONbored/gen-tests-4195-allowlist"; + const env = createTestEnv({ + GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), + AI: { run: async () => ({ response: "```typescript\n" + VALID_TEST_SOURCE + "\n```" }) } as unknown as Ai, + GITTENSORY_REVIEW_E2E_TESTS: "true", + GITTENSORY_REVIEW_REPOS: repoFullName, + AI_SUMMARIES_ENABLED: "true", + AI_PUBLIC_COMMENTS_ENABLED: "true", + }); + const slash = repoFullName.indexOf("/"); + await upsertRepositoryFromGitHub(env, { name: repoFullName.slice(slash + 1), full_name: repoFullName, private: false, owner: { login: repoFullName.slice(0, slash) } }, 123); + await upsertRepositorySettings(env, { repoFullName, commentMode: "off", publicSurface: "off", autoLabelEnabled: false, checkRunMode: "off", gateCheckMode: "enabled", requireLinkedIssue: false, linkedIssueGateMode: "advisory", aiReviewMode: "advisory" }); + await upsertPullRequestFromGitHub(env, repoFullName, { number: 4205, title: "Add retry to checkout", state: "open", user: { login: "contributor" }, author_association: "CONTRIBUTOR", head: { sha: "gen-tests-4195-allowlist" }, labels: [], body: "x" }); + await upsertPullRequestFile(env, { repoFullName, pullNumber: 4205, path: "src/checkout.ts", status: "modified", additions: 3, deletions: 0, changes: 3, payload: { patch: "+function retryPayment() {\n+ return true;\n+}" } }); + let postedBody = ""; + vi.stubGlobal("fetch", async (input: RequestInfo | URL, init?: RequestInit) => { + const url = input.toString(); + const method = init?.method ?? "GET"; + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + if (url.includes("/collaborators/maintainer/permission")) return Response.json({ permission: "admin" }); + if (url.includes("/issues/4205/comments") && method === "GET") return Response.json([]); + if (url.includes("/issues/4205/comments") && method === "POST") { postedBody = String((JSON.parse(String(init?.body ?? "{}")) as { body?: string }).body ?? ""); return Response.json({ id: 42050 }); } + return new Response("not found", { status: 404 }); + }); + + await processJob(env, generateTestsWebhook(repoFullName, 4205, "maintainer", { association: "MEMBER" })); + + expect(postedBody).toContain("test('checkout retries on failure'"); + }); + + it("skips cleanly when the webhook payload has no comment object at all", async () => { + const repoFullName = "JSONbored/gen-tests-4195-nocomment"; + const env = createTestEnv({ GITHUB_APP_PRIVATE_KEY: await generatePrivateKeyPem(), GITTENSORY_REVIEW_E2E_TESTS: "true", AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); + await seedGenerateTestsPr(env, repoFullName, 4206, "gen-tests-4195-nocomment"); + vi.stubGlobal("fetch", async (input: RequestInfo | URL) => { + const url = input.toString(); + if (url.includes("/access_tokens")) return Response.json({ token: "installation-token" }); + return new Response("not found", { status: 404 }); + }); + const webhook = generateTestsWebhook(repoFullName, 4206, "maintainer", { association: "MEMBER" }); + delete (webhook as unknown as { payload: { comment?: unknown } }).payload.comment; + + await processJob(env, webhook); + + const rows = await env.DB.prepare("select count(*) as n from audit_events where event_type like 'github_app.e2e_tests_generation%'").first<{ n: number }>(); + expect(rows?.n).toBe(0); + }); + }); + it("ops-alerts job no-ops when GITTENSORY_REVIEW_OPS is OFF (does no anomaly scan)", async () => { const env = createTestEnv(); // flag unset → OFF await env.DB.prepare("INSERT INTO repositories (full_name, owner, name, is_installed, is_registered) VALUES (?, ?, ?, 1, 1)")