diff --git a/docs.json b/docs.json index ca799fa..8c03443 100644 --- a/docs.json +++ b/docs.json @@ -350,8 +350,9 @@ "reference/mcp-server/tools/manage-proxies", "reference/mcp-server/tools/manage-extensions", "reference/mcp-server/tools/manage-apps", - "reference/mcp-server/tools/computer-action", "reference/mcp-server/tools/execute-playwright-code", + "reference/mcp-server/tools/screenshot", + "reference/mcp-server/tools/computer-action", "reference/mcp-server/tools/manage-replays", "reference/mcp-server/tools/exec-command", "reference/mcp-server/tools/search-docs" diff --git a/integrations/vercel/eve-extension.mdx b/integrations/vercel/eve-extension.mdx index 77b0b81..dcb23f8 100644 --- a/integrations/vercel/eve-extension.mdx +++ b/integrations/vercel/eve-extension.mdx @@ -66,7 +66,8 @@ Once mounted, the agent has the following tools, namespaced under your mount (e. - **`manage_browsers`**: create, list, get, and delete browser sessions. Returns a `session_id` and a `live_view_url` you can watch or take over. - **`execute_playwright_code`**: run Playwright against the live page to read, navigate, click, or type. -- **`computer_action`**: human-like mouse, keyboard, and screenshot controls for the same session. +- **`screenshot`**: capture what the session currently displays, so the agent can see the page. +- **`computer_action`**: human-like mouse, keyboard, and clipboard input for the same session, for surfaces Playwright selectors can't reach. - **`manage_auth_connections`**: Kernel's [managed auth](/auth/overview), so the agent logs into sites through a stored connection or a hosted login flow instead of typing credentials into the page. - **`manage_profiles`**: create and reuse browser [profiles](/auth/profiles) (persistent cookies, logins, storage). - **`manage_proxies`**: create and attach [proxies](/proxies/overview) (datacenter, ISP, residential, mobile) with geo-targeting. @@ -168,6 +169,7 @@ export default defineMcpClientConnection({ allow: [ "manage_browsers", "execute_playwright_code", + "screenshot", "computer_action", "browser_curl", // high blast radius: raw HTTP through the session "manage_auth_connections", diff --git a/reference/mcp-server/tools/computer-action.mdx b/reference/mcp-server/tools/computer-action.mdx index 9f73010..b801072 100644 --- a/reference/mcp-server/tools/computer-action.mdx +++ b/reference/mcp-server/tools/computer-action.mdx @@ -1,11 +1,13 @@ --- title: "computer_action" -description: "Mouse, keyboard, and screenshot controls for browser sessions" +description: "Mouse, keyboard, and clipboard input at screen coordinates" --- -Execute computer actions on a browser session. Pass a single action for simple operations, or pass multiple actions to batch them into one request for lower latency. +Drive a browser session with raw mouse and keyboard input at screen coordinates. Pass a single action for simple operations, or pass multiple actions to batch them into one request for lower latency. -Always include a `screenshot` as the last action so you can see the result. `screenshot` and `get_mouse_position` return data, so they must come last. +Prefer [`execute_playwright_code`](/reference/mcp-server/tools/execute-playwright-code) for anything a selector can reach. It's faster, deterministic, and doesn't depend on the model's ability to locate a target in an image. Reach for `computer_action` only when there's no selector to target: canvas apps, embedded PDFs, native dialogs, and drag interactions. + +Coordinates come from a [`screenshot`](/reference/mcp-server/tools/screenshot) call, and are only as accurate as the calling model's pixel grounding — models vary widely here. Take a screenshot after acting to confirm the result. ## Parameters @@ -26,9 +28,12 @@ Execute computer actions on a browser session. Pass a single action for simple o | `drag_mouse` | Drag along a `path` of `[x, y]` points. | | `set_cursor` | Show or hide the cursor (`hidden`). | | `sleep` | Wait `duration_ms` between steps when the page needs time to react. | -| `screenshot` | Capture the page, optionally limited to a `region`. | +| `write_clipboard` | Write `text` to the clipboard. | +| `read_clipboard` | Return the current clipboard contents. | | `get_mouse_position` | Return the current cursor position. | +`read_clipboard` and `get_mouse_position` return data, so they must come last. To capture the page, call the separate [`screenshot`](/reference/mcp-server/tools/screenshot) tool. + ## Example ```json @@ -38,8 +43,9 @@ Execute computer actions on a browser session. Pass a single action for simple o { "type": "click_mouse", "click_mouse": { "x": 420, "y": 300 } }, { "type": "type_text", "type_text": { "text": "kernel browsers" } }, { "type": "press_key", "press_key": { "keys": ["Return"] } }, - { "type": "sleep", "sleep": { "duration_ms": 1000 } }, - { "type": "screenshot" } + { "type": "sleep", "sleep": { "duration_ms": 1000 } } ] } ``` + +Then call `screenshot` to see the result. diff --git a/reference/mcp-server/tools/execute-playwright-code.mdx b/reference/mcp-server/tools/execute-playwright-code.mdx index ae15865..922f735 100644 --- a/reference/mcp-server/tools/execute-playwright-code.mdx +++ b/reference/mcp-server/tools/execute-playwright-code.mdx @@ -3,11 +3,11 @@ title: "execute_playwright_code" description: "Run Playwright/TypeScript code against a browser session" --- -Execute Playwright/TypeScript automation code against an existing Kernel browser session. This tool is a thin passthrough: it runs your code in the browser's VM and returns the result. It does not manage browser lifecycle — create and delete sessions with [`manage_browsers`](/reference/mcp-server/tools/manage-browsers). +Execute Playwright/TypeScript automation code against an existing Kernel browser session. This is the primary way to drive a browser: navigation, clicks, form fills, and extraction all belong here rather than in raw coordinate input. This tool is a thin passthrough: it runs your code in the browser's VM and returns the result. It does not manage browser lifecycle — create and delete sessions with [`manage_browsers`](/reference/mcp-server/tools/manage-browsers). `session_id` is required. Unlike earlier versions, this tool no longer creates a browser when `session_id` is omitted, and no longer deletes the browser after execution. Create a session with `manage_browsers` (action `create`), pass its `session_id` here, then delete it with `manage_browsers` when done. -Use `computer_action` with the `screenshot` action instead of `page.screenshot()` in your code. To read page state, return only what you need — prefer a targeted selector (e.g. `await page.locator('h1').innerText()`) or a region-scoped accessibility snapshot (e.g. `await page.locator('main').ariaSnapshot()`) rather than dumping the whole page. +Use the [`screenshot`](/reference/mcp-server/tools/screenshot) tool instead of `page.screenshot()` in your code. To read page state, return only what you need — prefer a targeted selector (e.g. `await page.locator('h1').innerText()`) or a region-scoped accessibility snapshot (e.g. `await page.locator('main').ariaSnapshot()`) rather than dumping the whole page. An `ariaSnapshot()` gives you the accessibility tree, which is usually a better way to locate an element than reading pixels off a screenshot. ## Parameters diff --git a/reference/mcp-server/tools/screenshot.mdx b/reference/mcp-server/tools/screenshot.mdx new file mode 100644 index 0000000..ef6cc13 --- /dev/null +++ b/reference/mcp-server/tools/screenshot.mdx @@ -0,0 +1,52 @@ +--- +title: "screenshot" +description: "Capture what a browser session currently displays" +--- + +Capture a PNG screenshot of a browser session. This tool is read-only: it observes the session without changing it. Use it to see page state, confirm what an automation did, or diagnose a flow that's stuck. + +Your MCP client must pass image content from tool results through to the model. Most clients do, but a text-only model can't read the screenshot even though the call succeeds. + +## Parameters + +| Parameter | Description | +|-----------|-------------| +| `session_id` | Browser session ID. Required. | +| `region` | Crop to a screen region: `x`, `y`, `width`, `height`. Omit to capture the full screen. | + +## Response + +Two content blocks: a text block describing the coordinate space, and the PNG image. + +For a full-screen capture, image coordinates are screen coordinates, so you can pass them straight to [`computer_action`](/reference/mcp-server/tools/computer-action): + +``` +Full screen 1024x768. Image coordinates are screen coordinates. +``` + +For a cropped capture, the image starts at the crop, so add the offset before using a coordinate: + +``` +Cropped region 400x200 at screen offset (100, 50). Image coordinates start at the +crop, so add the offset to get screen coordinates: screen_x = 100 + image_x, +screen_y = 50 + image_y. +``` + +## Example + +```json +{ + "session_id": "browser_abc123" +} +``` + +Crop to a region: + +```json +{ + "session_id": "browser_abc123", + "region": { "x": 100, "y": 50, "width": 400, "height": 200 } +} +``` + +To act on the page, prefer [`execute_playwright_code`](/reference/mcp-server/tools/execute-playwright-code) — selectors are faster and more reliable than clicking at coordinates read off a screenshot.