diff --git a/README.md b/README.md index 6529328..2dffeff 100644 --- a/README.md +++ b/README.md @@ -278,9 +278,10 @@ Self-hosted deployments can hide sensitive tool families by setting `KERNEL_MCP_ ### Standalone tools -- `computer_action` - Mouse, keyboard, clipboard, and screenshot controls for browser sessions (click, type, press_key, scroll, move, get_position, read_clipboard, write_clipboard, screenshot). +- `execute_playwright_code` - Execute Playwright/TypeScript code against an existing browser session. The primary way to drive a browser. Does not create or delete browsers - use `manage_browsers` for session lifecycle. +- `screenshot` - Capture a PNG of what a browser session currently displays, optionally cropped to a region. Read-only. +- `computer_action` - Mouse, keyboard, and clipboard input at screen coordinates (click, type, press_key, scroll, move, drag, get_position, read_clipboard, write_clipboard). Fallback for surfaces Playwright selectors can't reach, such as canvas apps and embedded PDFs; it depends on the model being able to locate targets in a screenshot, so prefer `execute_playwright_code`. - `browser_curl` - Send HTTP requests through an existing browser session's Chrome network stack. -- `execute_playwright_code` - Execute Playwright/TypeScript code against an existing browser session. Does not create or delete browsers - use `manage_browsers` for session lifecycle. - `exec_command` - Run shell commands inside a browser VM. Returns decoded stdout/stderr. - `search_docs` - Search Kernel platform documentation and guides. diff --git a/src/lib/mcp/prompts.ts b/src/lib/mcp/prompts.ts index 513e9fe..69f68ba 100644 --- a/src/lib/mcp/prompts.ts +++ b/src/lib/mcp/prompts.ts @@ -129,7 +129,7 @@ kernel browsers process --help kernel browsers playwright --help \`\`\` -**MCP Exceptions:** The \`computer_action\` MCP tool with action "screenshot" is useful since it returns images directly to the agent, and \`manage_browsers\` with action "get_telemetry" reads structured telemetry events (see below). +**MCP Exceptions:** The \`screenshot\` MCP tool is useful since it returns images directly to the agent, and \`manage_browsers\` with action "get_telemetry" reads structured telemetry events (see below). --- @@ -152,7 +152,7 @@ ${TELEMETRY_EVENT_CATALOG} kernel browsers get ${session_id} \`\`\` -### Take a screenshot (or use MCP computer_action with action "screenshot") +### Take a screenshot (or use the MCP screenshot tool) \`\`\`bash kernel browsers screenshot ${session_id} \`\`\` diff --git a/src/lib/mcp/register.ts b/src/lib/mcp/register.ts index 0b810a1..ab5cf11 100644 --- a/src/lib/mcp/register.ts +++ b/src/lib/mcp/register.ts @@ -16,6 +16,7 @@ import { registerProfileCapabilities } from "@/lib/mcp/tools/profiles"; import { registerProjectCapabilities } from "@/lib/mcp/tools/projects"; import { registerProxyTools } from "@/lib/mcp/tools/proxies"; import { registerReplayTools } from "@/lib/mcp/tools/replays"; +import { registerScreenshotTool } from "@/lib/mcp/tools/screenshot"; import { registerShellTool } from "@/lib/mcp/tools/shell"; type RegisterMcpToolset = (server: McpServer) => void; @@ -31,6 +32,7 @@ const mcpToolRegistrations = [ ["proxies", registerProxyTools], ["extensions", registerExtensionTools], ["apps", registerAppCapabilities], + ["screenshot", registerScreenshotTool], ["computer", registerComputerActionTool], ["shell", registerShellTool], ["playwright", registerPlaywrightTool], diff --git a/src/lib/mcp/tools/browser-pools.ts b/src/lib/mcp/tools/browser-pools.ts index 20ced89..8a095c4 100644 --- a/src/lib/mcp/tools/browser-pools.ts +++ b/src/lib/mcp/tools/browser-pools.ts @@ -451,7 +451,7 @@ export function registerBrowserPoolCapabilities(server: McpServer) { return jsonResponse({ browser: summarizeAcquiredBrowser(browser), next_actions: [ - `Use computer_action with session_id "${browser.session_id}" to control this browser.`, + `Use execute_playwright_code with session_id "${browser.session_id}" to drive this browser, and screenshot to see what it currently shows.`, `When finished, use manage_browser_pools with action "release", id_or_name "${poolId}", and session_id "${browser.session_id}".`, `Use manage_browsers with action "get" and session_id "${browser.session_id}" for full browser details.`, ], diff --git a/src/lib/mcp/tools/browsers.ts b/src/lib/mcp/tools/browsers.ts index 6865fe2..745a1a9 100644 --- a/src/lib/mcp/tools/browsers.ts +++ b/src/lib/mcp/tools/browsers.ts @@ -266,7 +266,7 @@ async function readBrowserTelemetry( function browserSessionNextActions(sessionId: string) { return [ - `Use computer_action with session_id "${sessionId}" to inspect or control the browser.`, + `Use execute_playwright_code with session_id "${sessionId}" to drive the browser, and screenshot to see what it currently shows.`, `Use manage_browsers with action "get" and session_id "${sessionId}" for full browser details.`, `Use manage_browsers with action "delete" and session_id "${sessionId}" when the session is no longer needed.`, ]; diff --git a/src/lib/mcp/tools/computer-action.ts b/src/lib/mcp/tools/computer-action.ts index 58291da..a36c10f 100644 --- a/src/lib/mcp/tools/computer-action.ts +++ b/src/lib/mcp/tools/computer-action.ts @@ -26,7 +26,6 @@ const computerActionSchema = z.object({ "sleep", "write_clipboard", "read_clipboard", - "screenshot", "get_mouse_position", ]) .describe("Action type."), @@ -107,26 +106,11 @@ const computerActionSchema = z.object({ }) .describe("Params for write_clipboard action.") .optional(), - screenshot: z - .object({ - region: z - .object({ - x: z.number(), - y: z.number(), - width: z.number().int().min(1), - height: z.number().int().min(1), - }) - .optional(), - }) - .describe( - "Params for screenshot action. Omit or pass {} for full-page screenshot.", - ) - .optional(), }); type ComputerActionParams = z.infer; type TerminalAction = ComputerActionParams & { - type: "screenshot" | "get_mouse_position" | "read_clipboard"; + type: "get_mouse_position" | "read_clipboard"; }; type WriteClipboardAction = ComputerActionParams & { type: "write_clipboard" }; type PrefixExecutionResult = @@ -137,9 +121,7 @@ function isTerminalAction( action: ComputerActionParams | undefined, ): action is TerminalAction { return ( - action?.type === "screenshot" || - action?.type === "get_mouse_position" || - action?.type === "read_clipboard" + action?.type === "get_mouse_position" || action?.type === "read_clipboard" ); } @@ -250,7 +232,7 @@ export function registerComputerActionTool(server: McpServer) { // computer_action -- Execute one or more computer actions on a browser session server.tool( "computer_action", - "Execute computer actions on a browser session. Pass a single action for simple operations (e.g. one click or one screenshot), or pass multiple actions to batch them into a single request for lower latency (e.g. click, type, press_key in one call). Use sleep actions between steps when the page needs time to react (e.g. after a click that triggers navigation or animation). IMPORTANT: Always include a screenshot as the last action so you can see the result of your actions. Action types: click_mouse, move_mouse, type_text, press_key, scroll, drag_mouse, set_cursor, sleep, write_clipboard, read_clipboard, screenshot, get_mouse_position. screenshot, read_clipboard, and get_mouse_position return data, so they must be the last action if included.", + "Drive a browser session with raw mouse and keyboard input at screen coordinates. Prefer execute_playwright_code for anything a selector can reach -- it is faster, deterministic, and does not depend on the model's ability to locate targets in an image. Reach for this tool only when there is no selector to target: canvas apps, embedded PDFs, native dialogs, drag interactions. Coordinates come from a screenshot tool call, and screen coordinates are only as accurate as the model's pixel grounding, so verify with screenshot after acting. Pass a single action, or several to batch them into one request for lower latency (e.g. click, type, press_key). Use sleep actions between steps when the page needs time to react. Action types: click_mouse, move_mouse, type_text, press_key, scroll, drag_mouse, set_cursor, sleep, write_clipboard, read_clipboard, get_mouse_position. read_clipboard and get_mouse_position return data, so they must be the last action if included.", { session_id: z.string().describe("Browser session ID."), actions: z @@ -261,7 +243,7 @@ export function registerComputerActionTool(server: McpServer) { ), }, { - title: "Control browser (mouse, keyboard, screenshot)", + title: "Control browser (mouse, keyboard)", readOnlyHint: false, destructiveHint: true, idempotentHint: false, @@ -288,45 +270,6 @@ export function registerComputerActionTool(server: McpServer) { const { executedActionCount } = prefixResult; - if (terminalAction?.type === "screenshot") { - const screenshotParams = terminalAction.screenshot; - const screenshotOpts = screenshotParams?.region - ? { region: screenshotParams.region } - : undefined; - const [screenshotResponse, browserInfo] = await Promise.all([ - client.browsers.computer.captureScreenshot( - session_id, - screenshotOpts, - ), - client.browsers.retrieve(session_id), - ]); - const blob = await screenshotResponse.blob(); - const buffer = Buffer.from(await blob.arrayBuffer()); - const viewport = browserInfo.viewport; - const content: Array< - | { type: "text"; text: string } - | { type: "image"; data: string; mimeType: string } - > = []; - if (executedActionCount > 0) { - content.push({ - type: "text", - text: `Executed ${executedActionCount} action(s), then captured screenshot.`, - }); - } - content.push({ - type: "text", - text: viewport - ? `Viewport: ${viewport.width}x${viewport.height}. Use these dimensions as the coordinate space for click, scroll, and move actions.` - : "Could not determine viewport dimensions. Use manage_browsers with action 'get' to check the browser's viewport.", - }); - content.push({ - type: "image", - data: buffer.toString("base64"), - mimeType: "image/png", - }); - return { content }; - } - if (terminalAction?.type === "get_mouse_position") { const position = await client.browsers.computer.getMousePosition(session_id); diff --git a/src/lib/mcp/tools/playwright.ts b/src/lib/mcp/tools/playwright.ts index 3e1a266..a3a582f 100644 --- a/src/lib/mcp/tools/playwright.ts +++ b/src/lib/mcp/tools/playwright.ts @@ -6,7 +6,7 @@ export function registerPlaywrightTool(server: McpServer) { // execute_playwright_code -- Run Playwright/TypeScript code against a browser server.tool( "execute_playwright_code", - "Execute Playwright/TypeScript automation code against an existing Kernel browser session. Does not create or delete browsers -- use manage_browsers to manage session lifecycle.", + "Execute Playwright/TypeScript automation code against an existing Kernel browser session. This is the primary way to drive a browser: navigation, clicks, form fills, and extraction should all go through here rather than raw coordinate input, and `await page.locator('main').ariaSnapshot()` gives you the accessibility tree to locate elements without needing a screenshot. Does not create or delete browsers -- use manage_browsers to manage session lifecycle.", { code: z .string() diff --git a/src/lib/mcp/tools/screenshot.ts b/src/lib/mcp/tools/screenshot.ts new file mode 100644 index 0000000..7b47165 --- /dev/null +++ b/src/lib/mcp/tools/screenshot.ts @@ -0,0 +1,89 @@ +import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { z } from "zod"; +import { createKernelClient } from "@/lib/mcp/kernel-client"; +import { toolErrorResponse } from "@/lib/mcp/responses"; + +const regionSchema = z.object({ + x: z.number(), + y: z.number(), + width: z.number().int().min(1), + height: z.number().int().min(1), +}); + +type Region = z.infer; + +async function coordinateSpaceText( + client: ReturnType, + sessionId: string, + region: Region | undefined, +) { + if (region) { + // The image is cropped, so its top-left is the region origin, not the screen + // origin. Computer actions take screen coordinates, so spell out the offset + // rather than leaving the model to guess which space it's looking at. + return `Cropped region ${region.width}x${region.height} at screen offset (${region.x}, ${region.y}). Image coordinates start at the crop, so add the offset to get screen coordinates: screen_x = ${region.x} + image_x, screen_y = ${region.y} + image_y.`; + } + + const { viewport } = await client.browsers.retrieve(sessionId); + if (!viewport) { + return "Could not determine viewport dimensions. Use manage_browsers with action 'get' to check the browser's viewport."; + } + + return `Full screen ${viewport.width}x${viewport.height}. Image coordinates are screen coordinates.`; +} + +export function registerScreenshotTool(server: McpServer) { + // screenshot -- Capture what a browser session currently shows + server.tool( + "screenshot", + "Capture a PNG screenshot of what a browser session currently displays. Read-only: it observes the session without changing it. Use it to see page state, confirm what an automation did, or diagnose a stuck flow. To act on the page, prefer execute_playwright_code.", + { + session_id: z + .string() + .min(1, "session_id is required") + .describe("Browser session ID."), + region: regionSchema + .describe( + "Crop to this screen region. Omit to capture the full screen.", + ) + .optional(), + }, + { + title: "Screenshot browser session", + readOnlyHint: true, + destructiveHint: false, + idempotentHint: true, + openWorldHint: true, + }, + async ({ session_id, region }, extra) => { + if (!extra.authInfo) throw new Error("Authentication required"); + const client = createKernelClient(extra.authInfo.token); + + try { + const [screenshotResponse, spaceText] = await Promise.all([ + client.browsers.computer.captureScreenshot( + session_id, + region ? { region } : undefined, + ), + coordinateSpaceText(client, session_id, region), + ]); + + const blob = await screenshotResponse.blob(); + const buffer = Buffer.from(await blob.arrayBuffer()); + + return { + content: [ + { type: "text" as const, text: spaceText }, + { + type: "image" as const, + data: buffer.toString("base64"), + mimeType: "image/png", + }, + ], + }; + } catch (error) { + return toolErrorResponse("screenshot", "capture", error); + } + }, + ); +}