Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 5 additions & 15 deletions src/components/BatchEvaluationPicker.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -41,11 +41,6 @@ function toRow(summary: BatchEvaluationSummary): BatchEvaluationRow {
export interface BatchEvaluationPickerProps extends ScreenProps {
breadcrumb: string[];
description?: string;
queryKeyPrefix?: string;
include?: (summary: BatchEvaluationSummary) => boolean;
loadingMessage?: string;
emptyMessage?: string;
emptyPageMessage?: string;
onSelect: (batchEvaluationId: string) => void;
onEscape?: () => void;
}
Expand All @@ -60,11 +55,6 @@ export function BatchEvaluationPicker({
core,
breadcrumb,
description,
queryKeyPrefix = "batch-evaluations",
include,
loadingMessage = "Loading batch evaluations…",
emptyMessage = "No batch evaluations found in this Region.",
emptyPageMessage = "No batch evaluations on this page.",
onSelect,
onEscape,
}: BatchEvaluationPickerProps) {
Expand All @@ -76,11 +66,11 @@ export function BatchEvaluationPicker({
<PaginatedTablePicker
breadcrumb={breadcrumb}
description={description}
queryKey={[queryKeyPrefix, opts.region]}
queryKey={["batch-evaluations", opts.region]}
loadPage={async (token, pageSize) => {
const response = await core.eval.listBatchEvaluations(token, pageSize, opts);
return {
items: (response.batchEvaluations ?? []).filter((summary) => include?.(summary) ?? true),
items: response.batchEvaluations ?? [],
nextToken: response.nextToken,
};
}}
Expand All @@ -89,10 +79,10 @@ export function BatchEvaluationPicker({
getValue={(row) => row.batchEvaluationId}
onSelect={onSelect}
onBack={goBack}
loadingMessage={loadingMessage}
loadingMessage="Loading batch evaluations…"
errorMessage={(error) => `Error: ${error.message}`}
emptyMessage={emptyMessage}
emptyPageMessage={emptyPageMessage}
emptyMessage="No batch evaluations found in this Region."
emptyPageMessage="No batch evaluations on this page."
/>
);
}
79 changes: 79 additions & 0 deletions src/components/BatchInsightsPicker.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
import type { BatchEvaluationSummary } from "@aws-sdk/client-bedrock-agentcore";
import { useNavigate } from "react-router";
import type { ScreenProps } from "../handlers/types";
import { coreOptsFromCtx } from "../handlers/utils";
import { formatTimestamp } from "./formatTimestamp";
import { PaginatedTablePicker } from "./PaginatedTablePicker";
import type { DataTableColumn } from "./ui/data-table";

interface BatchInsightsRow extends Record<string, unknown> {
batchEvaluationId: string;
name: string;
status: string;
updatedAt: string;
}

const batchInsightsColumns = [
{ key: "name", header: "name", flex: true },
{ key: "status", header: "status", width: 22 },
{
key: "updatedAt",
header: "updated UTC",
width: 16,
render: formatTimestamp,
},
] satisfies DataTableColumn<BatchInsightsRow>[];

function toRow(summary: BatchEvaluationSummary): BatchInsightsRow {
const id = summary.batchEvaluationId ?? "";
return {
batchEvaluationId: id,
name: summary.batchEvaluationName ?? id,
status: summary.status ?? "-",
updatedAt: summary.updatedAt?.toISOString() ?? "-",
};
}

export interface BatchInsightsPickerProps extends ScreenProps {
breadcrumb: string[];
description?: string;
onSelect: (batchEvaluationId: string) => void;
onEscape?: () => void;
}

export function BatchInsightsPicker({
ctx,
core,
breadcrumb,
description,
onSelect,
onEscape,
}: BatchInsightsPickerProps) {
const opts = coreOptsFromCtx(ctx);
const navigate = useNavigate();
const goBack = onEscape ?? (() => navigate("/" + breadcrumb.slice(0, -1).join("/")));

return (
<PaginatedTablePicker
breadcrumb={breadcrumb}
description={description}
queryKey={["batch-insights", opts.region]}
loadPage={async (token, pageSize) => {
const response = await core.eval.listBatchInsights(token, pageSize, opts);
return {
items: response.batchEvaluations ?? [],
nextToken: response.nextToken,
};
}}
toRow={toRow}
columns={batchInsightsColumns}
getValue={(row) => row.batchEvaluationId}
onSelect={onSelect}
onBack={goBack}
loadingMessage="Loading batch insights…"
errorMessage={(error) => `Error: ${error.message}`}
emptyMessage="No batch insights found in this Region."
emptyPageMessage="No batch insights on this page."
/>
);
}
210 changes: 210 additions & 0 deletions src/core/batchInsights.test.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
import { describe, expect, mock, test } from "bun:test";

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does gateway have a seperate unit test for this? Can this be tested within the handler tests?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

gateway tests this via unit tests also: src/core/gateway.test.ts:65

import {
GetBatchEvaluationCommand,
ListBatchEvaluationsCommand,
type BatchEvaluationSummary,
} from "@aws-sdk/client-bedrock-agentcore";
import { ResultTruncationError } from "../errors";
import type { AwsClients } from "./types";
import { EvalClient } from "./eval";

const options = { region: "us-west-2", endpointUrl: "https://agentcore.example.test" };

function insight(id: string): BatchEvaluationSummary {
return {
batchEvaluationId: id,
insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }],
} as BatchEvaluationSummary;
}

function evaluation(id: string): BatchEvaluationSummary {
return {
batchEvaluationId: id,
evaluators: [{ evaluatorId: "Builtin.Correctness" }],
} as BatchEvaluationSummary;
}

function evalClient(send: (command: ListBatchEvaluationsCommand) => Promise<unknown>): EvalClient {
return new EvalClient({
data: () => ({ send: mock(send) }) as never,
} as unknown as AwsClients);
}

function getEvalClient(send: (command: GetBatchEvaluationCommand) => Promise<unknown>): EvalClient {
return new EvalClient({
data: () => ({ send: mock(send) }) as never,
} as unknown as AwsClients);
}

describe("EvalClient.getBatchInsights", () => {
test("gets an Insights job without reading CloudWatch evaluation results", async () => {
const job = {
batchEvaluationId: "insights-1",
batchEvaluationArn:
"arn:aws:bedrock-agentcore:us-west-2:123456789012:batch-evaluate/insights-1",
batchEvaluationName: "insights-1",
status: "COMPLETED" as const,
createdAt: new Date("2026-08-25T12:00:00.000Z"),
insights: [{ insightId: "Builtin.Insight.FailureAnalysis" }],
outputConfig: {
cloudWatchConfig: {
logGroupName: "/aws/example",
logStreamName: "results",
},
},
};
const client = getEvalClient(async (command) => {
expect(command).toBeInstanceOf(GetBatchEvaluationCommand);
expect(command.input).toEqual({ batchEvaluationId: "insights-1" });
return job;
});

await expect(client.getBatchInsights("insights-1", options)).resolves.toEqual(job);
});

test("rejects an evaluator-only Batch Evaluation", async () => {
const client = getEvalClient(async () => evaluation("evaluation-1"));

await expect(client.getBatchInsights("evaluation-1", options)).rejects.toThrow(
'batch evaluation "evaluation-1" is not a batch insights run',
);
});
});

describe("EvalClient.listBatchInsights", () => {
test("rejects an invalid logical page size before calling the service", async () => {
const send = mock(async () => ({ batchEvaluations: [] }));
const client = evalClient(send);

await expect(client.listBatchInsights(undefined, 0, options)).rejects.toThrow(
"maxResults must be a positive integer",
);
expect(send).not.toHaveBeenCalled();
});

test("filters the final Batch Evaluation page", async () => {
const insightsJob = insight("insights-1");
const client = evalClient(async (command) => {
expect(command).toBeInstanceOf(ListBatchEvaluationsCommand);
expect(command.input).toEqual({ nextToken: "page-2", maxResults: undefined });
return { batchEvaluations: [evaluation("evaluation-1"), insightsJob] };
});

await expect(client.listBatchInsights("page-2", 10, options)).resolves.toEqual({
batchEvaluations: [insightsJob],
nextToken: undefined,
});
});

test("scans sparse service pages to fill one logical Insights page", async () => {
const insights = [insight("insights-1"), insight("insights-2")];
const requests: unknown[] = [];
const client = evalClient(async (command) => {
requests.push(command.input);
switch (command.input.nextToken) {
case undefined:
return {
batchEvaluations: [evaluation("evaluation-1")],
nextToken: "page-2",
};
case "page-2":
return {
batchEvaluations: [insights[0], evaluation("evaluation-2")],
nextToken: "page-3",
};
default:
return {
batchEvaluations: [evaluation("evaluation-3"), insights[1]],
};
}
});

await expect(client.listBatchInsights(undefined, 2, options)).resolves.toEqual({
batchEvaluations: insights,
nextToken: undefined,
});
expect(requests).toEqual([
{ nextToken: undefined, maxResults: undefined },
{ nextToken: "page-2", maxResults: undefined },
{ nextToken: "page-3", maxResults: undefined },
]);
});

test("replays the exact Batch Evaluation prefix without skipping the next Insight", async () => {
const firstInsight = insight("insights-1");
const secondInsight = insight("insights-2");
const requests: unknown[] = [];
const client = evalClient(async (command) => {
requests.push(command.input);

if (command.input.nextToken === "after-insights-1") {
return {
batchEvaluations: [evaluation("evaluation-2"), secondInsight],
};
}
if (command.input.maxResults === 2) {
return {
batchEvaluations: [evaluation("evaluation-1"), firstInsight],
nextToken: "after-insights-1",
};
}
return {
batchEvaluations: [
evaluation("evaluation-1"),
firstInsight,
evaluation("evaluation-2"),
secondInsight,
],
};
});

const first = await client.listBatchInsights(undefined, 1, options);
const second = await client.listBatchInsights(first.nextToken, 1, options);

expect(first).toEqual({
batchEvaluations: [firstInsight],
nextToken: "after-insights-1",
});
expect(second).toEqual({
batchEvaluations: [secondInsight],
nextToken: undefined,
});
expect(requests).toEqual([
{ nextToken: undefined, maxResults: undefined },
{ nextToken: undefined, maxResults: 2 },
{ nextToken: "after-insights-1", maxResults: undefined },
]);
});

test("returns a token only when it leads to another Insights job", async () => {
const firstInsight = insight("insights-1");
const secondInsight = insight("insights-2");
const client = evalClient(async (command) => {
if (command.input.nextToken === "page-2") {
return { batchEvaluations: [evaluation("evaluation-2"), secondInsight] };
}
return {
batchEvaluations: [firstInsight, evaluation("evaluation-1")],
nextToken: "page-2",
};
});

await expect(client.listBatchInsights(undefined, 1, options)).resolves.toEqual({
batchEvaluations: [firstInsight],
nextToken: "page-2",
});
});

test("throws when Insights discovery exceeds the Batch Evaluation scan cap", async () => {
let calls = 0;
const client = evalClient(async () => {
calls += 1;
return { batchEvaluations: [], nextToken: `page-${calls}` };
});

await expect(client.listBatchInsights(undefined, 1, options)).rejects.toThrow(
ResultTruncationError,
);
expect(calls).toBe(101);
});
});
Loading
Loading