Skip to content

fix(codex): show image generation tool calls #34020

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Yash-Singh1 merged 3 commits into from
Nov 10, 2026
+146 −4
Merged

fix(codex): show image generation tool calls #34020

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b04d3dd
fix(opencode): emit image generation tool calls
Yash-Singh1 Nov 10, 2026
7ba6f83
Merge remote-tracking branch 'origin/main' into t3/fix-image-generati…
Yash-Singh1 Nov 10, 2026
86d2045
fix(codex): retain image inputs when cancelling generation
Yash-Singh1 Nov 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 65 additions & 0 deletions apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4116,6 +4116,8 @@ describe("CodexAdapterV2 post-settle continuation", () => {
id: `image-${index}`,
status: "completed",
result,
revisedPrompt: "A tree shaped like T3",
transparentBackground: true,
...image,
},
},
Expand All @@ -4136,6 +4138,42 @@ describe("CodexAdapterV2 post-settle continuation", () => {
);
yield* harness.firstTerminal;

const imageItems = harness.events
.flatMap((event) => (event.type === "turn_item.updated" ? [event.turnItem] : []))
.filter((item) => item.type === "dynamic_tool" && item.toolName === "image_gen.imagegen");
const latestItems = new Map(
harness.events.flatMap((event) =>
event.type === "turn_item.updated" ? [[event.turnItem.id, event.turnItem] as const] : [],
),
);
assert.strictEqual(
[...latestItems.values()].filter((item) => item.type === "dynamic_tool").length,
images.length,
);
assert.strictEqual(imageItems.length, images.length);
assert.deepEqual(
imageItems.map((item) => item.status),
images.map((image) =>
("status" in image && image.status === "failed") || "failure" in image
? "failed"
: "completed",
),
);
for (const [index, item] of imageItems.entries()) {
assert.strictEqual(item.title, "Generate image");
if (item.type !== "dynamic_tool") continue;
assert.deepEqual(item.input, {
prompt: "A tree shaped like T3",
transparent_background: true,
});
const image = images[index]!;
assert.deepEqual(
item.output,
Object.fromEntries(
Object.entries({ status: "completed", ...image }).filter(([, value]) => value != null),
),
);
}
const messages = assistantMessages(harness.events);
assert.strictEqual(messages.at(-1)?.message.text, expected);
assert.strictEqual(messages.at(-1)?.message.streaming, false);
Expand Down Expand Up @@ -4179,6 +4217,8 @@ describe("CodexAdapterV2 post-settle continuation", () => {
item: {
type: "imageGeneration",
id: image,
revisedPrompt: "A friendly green tree",
transparentBackground: true,
status: phase === "started" ? "inProgress" : "completed",
result: "",
...(phase === "completed" ? { savedPath: `/tmp/${image}.png` } : {}),
Expand Down Expand Up @@ -4249,6 +4289,31 @@ describe("CodexAdapterV2 post-settle continuation", () => {
yield* Deferred.await(drained);
assert.deepEqual(yield* pending, [false, false]);
assert.lengthOf(harness.terminalEvents(), 1);
const imageItems = harness.events.flatMap((event) =>
event.type === "turn_item.updated" &&
event.turnItem.type === "dynamic_tool" &&
event.turnItem.toolName === "image_gen.imagegen"
? [event.turnItem]
: [],
);
assert.deepEqual(
imageItems.slice(0, 2).map((item) => item.status),
["running", "running"],
);
for (const id of new Set(imageItems.map((item) => item.id))) {
const updates = imageItems.filter((item) => item.id === id);
assert.lengthOf(updates, 2);
assert.strictEqual(
updates[1]?.status,
outcome === "complete" ? "completed" : "cancelled",
);
assert.strictEqual(updates[1]?.ordinal, updates[0]?.ordinal);
assert.deepEqual(updates[1]?.input, updates[0]?.input);
assert.deepEqual(updates[1]?.input, {
prompt: "A friendly green tree",
transparent_background: true,
});
}
// Run ingestion follows these updates, so they must bracket the terminal and the images.
const indexOf = (predicate: (event: ProviderAdapter.ProviderAdapterV2Event) => boolean) =>
harness.events.findIndex(predicate);
Expand Down
85 changes: 81 additions & 4 deletions apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1981,7 +1981,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
const imageTurns = new Map<
string,
{
readonly running: Set<string>;
readonly running: Map<
string,
{
readonly revisedPrompt: string | null;
readonly transparentBackground: boolean | null;
}
>;
readonly saved: Map<string, string>;
pending: boolean;
answer?: { readonly id: string; readonly text: string };
Expand All @@ -1990,7 +1996,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
const imageTurn = (nativeTurnId: string) => {
let state = imageTurns.get(nativeTurnId);
if (state === undefined) {
state = { running: new Set(), saved: new Map(), pending: false };
state = { running: new Map(), saved: new Map(), pending: false };
imageTurns.set(nativeTurnId, state);
}
return state;
Expand Down Expand Up @@ -2075,6 +2081,19 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
const clearImageWork = Effect.fn("CodexAdapterV2.clearImageWork")(function* (
context: ActiveCodexTurnContext,
) {
for (const [id, input] of imageTurns.get(context.nativeTurnId)?.running ?? []) {
yield* emitImageGenerationItem(
context,
{
Comment thread
macroscopeapp[bot] marked this conversation as resolved.
Resolved
type: "imageGeneration",
id,
...input,
status: "cancelled",
result: "",
},
true,
);
}
yield* emitImageWorkState(context, false);
imageTurns.delete(context.nativeTurnId);
});
Expand Down Expand Up @@ -3926,6 +3945,56 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
},
);

const emitImageGenerationItem = Effect.fn("CodexAdapterV2.emitImageGenerationItem")(
function* (
context: ActiveCodexTurnContext,
item: Extract<
CodexSchema.V2ItemCompletedNotification__ThreadItem,
{ type: "imageGeneration" }
>,
completed: boolean,
) {
const cancelled = completed && item.status === "cancelled";
const failed = item.status === "failed" || item.failure != null;
const artifacts = yield* buildDynamicToolArtifacts(context, {
type: "dynamicToolCall",
id: item.id,
namespace: "image_gen",
tool: "imagegen",
status: completed ? (failed ? "failed" : "completed") : "inProgress",
arguments: {
...(item.revisedPrompt == null ? {} : { prompt: item.revisedPrompt }),
...(item.transparentBackground == null
? {}
: { transparent_background: item.transparentBackground }),
},
});
yield* emitProviderEvent({
type: "node.updated",
driver: CODEX_PROVIDER,
node: { ...artifacts.node, ...(cancelled ? { status: "cancelled" } : {}) },
});
yield* emitProviderEvent({
type: "turn_item.updated",
driver: CODEX_PROVIDER,
turnItem: {
...artifacts.turnItem,
title: "Generate image",
...(cancelled ? { status: "cancelled" } : {}),
...(completed
? {
output: {
status: item.status,
...(item.savedPath == null ? {} : { savedPath: item.savedPath }),
...(item.failure == null ? {} : { failure: item.failure }),
},
}
: {}),
},
});
},
);

/**
* The item stays running until then, which keeps the run's event
* ingestion open past turn/completed; it runs off the notification
Expand Down Expand Up @@ -4980,7 +5049,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (

if (payload.item.type === "imageGeneration") {
if (!(yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) {
imageTurn(payload.turnId).running.add(payload.item.id);
imageTurn(payload.turnId).running.set(payload.item.id, {
revisedPrompt: payload.item.revisedPrompt ?? null,
transparentBackground: payload.item.transparentBackground ?? null,
});
yield* emitImageGenerationItem(context, payload.item, false);
yield* emitImageWorkState(context, true);
}
return;
Expand Down Expand Up @@ -5266,6 +5339,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (

if (payload.item.type === "imageGeneration") {
if ((yield* Ref.get(interruptingNativeTurns)).has(payload.turnId)) return;
yield* emitImageGenerationItem(context, payload.item, true);
const { id, savedPath, status, failure } = payload.item;
const state = imageTurn(payload.turnId);
if (savedPath?.trim() && status === "completed" && failure == null) {
Expand Down Expand Up @@ -6182,7 +6256,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* (
state.saved.clear();
return;
}
const message = { id: finalAnswer?.id ?? firstImageItemId, text };
const message = {
id: finalAnswer?.id ?? `generated-image-message:${firstImageItemId}`,
text,
};
state.answer = message;
const artifacts = yield* buildAgentMessageArtifacts(context, message, true);
yield* emitProviderEvent({
Expand Down
Loading