Skip to content

fix: never upscale captures, and name their image metadata image #69

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iamnbutler merged 1 commit into from
Oct 17, 2026
Merged

fix: never upscale captures, and name their image metadata image #69

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 21 additions & 14 deletions dist/client.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,28 +152,27 @@ async function screenshot(input, output, signal) {
if (!file.isFile() || file.size > 32_000_000) {
throw new Error("The native screenshot is missing or too large.");
}
const source = await dimensions(input, signal);
const longest = Math.max(source.width, source.height);
// A single result must fit the hosted channel's 2 MB SQLite row, including its base64 image.
for (const size of [1600, 1200, 800]) {
// Captures are only scaled down: a small display or window keeps its own pixels.
const sizes = [...new Set([1600, 1200, 800].map((size) => Math.min(size, longest)))];
for (const size of sizes) {
await run("/usr/bin/sips", [
"-s",
"format",
"jpeg",
"-s",
"formatOptions",
"70",
"--resampleHeightWidthMax",
String(size),
...(size < longest ? ["--resampleHeightWidthMax", String(size)] : []),
input,
"--out",
output,
], signal);
if ((await stat(output)).size > MAX_IMAGE)
continue;
const info = await run("/usr/bin/sips", ["-g", "pixelWidth", "-g", "pixelHeight", output], signal);
const width = Number(/pixelWidth:\s*(\d+)/.exec(info)?.[1]);
const height = Number(/pixelHeight:\s*(\d+)/.exec(info)?.[1]);
if (!width || !height)
throw new Error("Could not read the screenshot dimensions.");
const { width, height } = await dimensions(output, signal);
const bytes = await readFile(output);
if (bytes.length > MAX_IMAGE) {
throw new Error("The resized screenshot exceeds the result limit.");
Expand All @@ -182,6 +181,14 @@ async function screenshot(input, output, signal) {
}
throw new Error("The screenshot could not be resized within the result limit.");
}
async function dimensions(path, signal) {
const info = await run("/usr/bin/sips", ["-g", "pixelWidth", "-g", "pixelHeight", path], signal);
const width = Number(/pixelWidth:\s*(\d+)/.exec(info)?.[1]);
const height = Number(/pixelHeight:\s*(\d+)/.exec(info)?.[1]);
if (!width || !height)
throw new Error("Could not read the screenshot dimensions.");
return { width, height };
}
function clipboardFiles(data) {
const files = data.files;
if (typeof data.present !== "boolean" || !Number.isSafeInteger(data.change_count)
Expand Down Expand Up @@ -261,10 +268,10 @@ async function inspect(native, request, signal) {
const { screenshot_raw: _raw, screenshot_annotated: _annotated, ...observation } = data;
const text = bounded({
...observation,
ace_image: {
image: {
width,
height,
note: "Screenshot resized; Accessibility bounds remain in their original coordinate system. Pointer points use fractions of this image: x from the left edge and y from the top, each >= 0 and < 1.",
note: "Screenshot, scaled down when large but never up; Accessibility bounds remain in their original coordinate system. Pointer points use fractions of this image: x from the left edge and y from the top, each >= 0 and < 1.",
},
}, "ui_elements", limit);
return { text, image };
Expand Down Expand Up @@ -311,10 +318,10 @@ async function screen(native, request, signal) {
const { image, width, height } = await screenshot(path, join(directory, "image.jpg"), signal);
const text = bounded({
...data,
ace_image: {
image: {
width,
height,
note: "Whole-display screenshot, resized. Observation only: there is no snapshot_id and no input can target this image. Window and display bounds are global desktop points; pass a window's pid and window_id to desktop_inspect to act on it.",
note: "Whole-display screenshot, scaled down when large but never up. Observation only: there is no snapshot_id and no input can target this image. Window and display bounds are global desktop points; pass a window's pid and window_id to desktop_inspect to act on it.",
},
}, "windows", limit);
return { text, image };
Expand Down Expand Up @@ -996,10 +1003,10 @@ async function menubarClick(native, request, signal) {
const { image, width, height } = await screenshot(join(directory, "capture.png"), join(directory, "image.jpg"), signal);
const text = JSON.stringify({
action: data,
ace_image: {
image: {
width,
height,
note: "Screenshot of the region that opened after the click, resized. Observation only: there is no snapshot_id and no input can target this image.",
note: "Screenshot of the region that opened after the click, scaled down when large but never up. Observation only: there is no snapshot_id and no input can target this image.",
},
});
if (Buffer.byteLength(text) > MAX_TEXT)
Expand Down
38 changes: 21 additions & 17 deletions src/client.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,30 +223,26 @@ async function screenshot(input: string, output: string, signal: AbortSignal) {
if (!file.isFile() || file.size > 32_000_000) {
throw new Error("The native screenshot is missing or too large.");
}
const source = await dimensions(input, signal);
const longest = Math.max(source.width, source.height);
// A single result must fit the hosted channel's 2 MB SQLite row, including its base64 image.
for (const size of [1600, 1200, 800]) {
// Captures are only scaled down: a small display or window keeps its own pixels.
const sizes = [...new Set([1600, 1200, 800].map((size) => Math.min(size, longest)))];
for (const size of sizes) {
await run("/usr/bin/sips", [
"-s",
"format",
"jpeg",
"-s",
"formatOptions",
"70",
"--resampleHeightWidthMax",
String(size),
...(size < longest ? ["--resampleHeightWidthMax", String(size)] : []),
input,
"--out",
output,
], signal);
if ((await stat(output)).size > MAX_IMAGE) continue;
const info = await run(
"/usr/bin/sips",
["-g", "pixelWidth", "-g", "pixelHeight", output],
signal,
);
const width = Number(/pixelWidth:\s*(\d+)/.exec(info)?.[1]);
const height = Number(/pixelHeight:\s*(\d+)/.exec(info)?.[1]);
if (!width || !height) throw new Error("Could not read the screenshot dimensions.");
const { width, height } = await dimensions(output, signal);
const bytes = await readFile(output);
if (bytes.length > MAX_IMAGE) {
throw new Error("The resized screenshot exceeds the result limit.");
Expand All @@ -256,6 +252,14 @@ async function screenshot(input: string, output: string, signal: AbortSignal) {
throw new Error("The screenshot could not be resized within the result limit.");
}

async function dimensions(path: string, signal: AbortSignal) {
const info = await run("/usr/bin/sips", ["-g", "pixelWidth", "-g", "pixelHeight", path], signal);
const width = Number(/pixelWidth:\s*(\d+)/.exec(info)?.[1]);
const height = Number(/pixelHeight:\s*(\d+)/.exec(info)?.[1]);
if (!width || !height) throw new Error("Could not read the screenshot dimensions.");
return { width, height };
}

function clipboardFiles(data: Record<string, unknown>): DesktopResult {
const files = data.files;
if (
Expand Down Expand Up @@ -344,11 +348,11 @@ async function inspect(
const { screenshot_raw: _raw, screenshot_annotated: _annotated, ...observation } = data;
const text = bounded({
...observation,
ace_image: {
image: {
width,
height,
note:
"Screenshot resized; Accessibility bounds remain in their original coordinate system. Pointer points use fractions of this image: x from the left edge and y from the top, each >= 0 and < 1.",
"Screenshot, scaled down when large but never up; Accessibility bounds remain in their original coordinate system. Pointer points use fractions of this image: x from the left edge and y from the top, each >= 0 and < 1.",
},
}, "ui_elements", limit);
return { text, image };
Expand Down Expand Up @@ -395,11 +399,11 @@ async function screen(
const { image, width, height } = await screenshot(path, join(directory, "image.jpg"), signal);
const text = bounded({
...data,
ace_image: {
image: {
width,
height,
note:
"Whole-display screenshot, resized. Observation only: there is no snapshot_id and no input can target this image. Window and display bounds are global desktop points; pass a window's pid and window_id to desktop_inspect to act on it.",
"Whole-display screenshot, scaled down when large but never up. Observation only: there is no snapshot_id and no input can target this image. Window and display bounds are global desktop points; pass a window's pid and window_id to desktop_inspect to act on it.",
},
}, "windows", limit);
return { text, image };
Expand Down Expand Up @@ -1184,11 +1188,11 @@ async function menubarClick(
);
const text = JSON.stringify({
action: data,
ace_image: {
image: {
width,
height,
note:
"Screenshot of the region that opened after the click, resized. Observation only: there is no snapshot_id and no input can target this image.",
"Screenshot of the region that opened after the click, scaled down when large but never up. Observation only: there is no snapshot_id and no input can target this image.",
},
});
if (Buffer.byteLength(text) > MAX_TEXT) return actionResult(data, outcome);
Expand Down
Loading