diff --git a/src/core/eval.tsx b/src/core/eval.tsx index 6480299d1..7c5032bd5 100644 --- a/src/core/eval.tsx +++ b/src/core/eval.tsx @@ -76,6 +76,7 @@ import { type EvaluationReferenceInput, type EvaluationResultContent, type EvaluationTarget, + type CreateABTestRequest, type CreateABTestResponse, type GetABTestResponse, type ListABTestsResponse, @@ -124,6 +125,7 @@ import type { CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBasedABTestInput, + CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -491,65 +493,38 @@ export class EvalClient implements CoreEvalClient { .send(new DeleteABTestCommand({ abTestId: id })); } - async createConfigBasedABTest( - input: CreateConfigBasedABTestInput, + private async createABTest( + name: string, + gateway: string, + callerRoleArn: string | undefined, + build: (context: { + gatewayArn: string; + accountId: string; + roleArn: string; + }) => CreateABTestRequest, options: CoreOptions, ): Promise { const control = this.clients.control(toClientConfig(options)); - const gateway = await control.send(new GetGatewayCommand({ gatewayIdentifier: input.gateway })); - const gatewayArn = gateway.gatewayArn!; + const gatewayArn = (await control.send(new GetGatewayCommand({ gatewayIdentifier: gateway }))) + .gatewayArn!; const accountId = accountIdFromArn(gatewayArn); - const controlBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.control.configBundle}`; - const treatmentBundleArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${input.treatment.configBundle}`; - const onlineEvaluationConfigArn = `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`; - - const treatmentWeight = input.treatmentWeight ?? 50; - const variants = [ - { - name: "C", - weight: 100 - treatmentWeight, - variantConfiguration: { - configurationBundle: { - bundleArn: controlBundleArn, - bundleVersion: input.control.bundleVersion, - }, - }, - }, - { - name: "T1", - weight: treatmentWeight, - variantConfiguration: { - configurationBundle: { - bundleArn: treatmentBundleArn, - bundleVersion: input.treatment.bundleVersion, - }, - }, - }, - ]; - - let roleArn = input.roleArn; + let roleArn = callerRoleArn; let provisionedRoleArn: string | undefined; if (!roleArn) { - const iam = this.clients.iam({ region: options.region }); - const provisioned = await provisionAbTestRole(iam, input.name, gatewayArn, options.region); + const provisioned = await provisionAbTestRole( + this.clients.iam({ region: options.region }), + name, + gatewayArn, + options.region, + ); roleArn = provisioned.roleArn; if (provisioned.created) provisionedRoleArn = provisioned.roleArn; } - const command = new CreateABTestCommand({ - name: input.name, - gatewayArn, - variants, - evaluationConfig: { onlineEvaluationConfigArn }, - roleArn, - gatewayFilter: input.gatewayFilter, - enableOnCreate: input.enableOnCreate ?? true, - clientToken: randomUUID(), - }); - + const command = new CreateABTestCommand(build({ gatewayArn, accountId, roleArn })); try { - return input.roleArn + return callerRoleArn ? await this.clients.data(toClientConfig(options)).send(command) : await retryWhileRolePropagates(() => this.clients.data(toClientConfig(options)).send(command), @@ -566,6 +541,99 @@ export class EvalClient implements CoreEvalClient { } } + async createConfigBasedABTest( + input: CreateConfigBasedABTestInput, + options: CoreOptions, + ): Promise { + const treatmentWeight = input.treatmentWeight ?? 50; + return this.createABTest( + input.name, + input.gateway, + input.roleArn, + ({ gatewayArn, accountId, roleArn }) => { + const bundleArn = (id: string) => + `arn:aws:bedrock-agentcore:${options.region}:${accountId}:configuration-bundle/${id}`; + return { + name: input.name, + gatewayArn, + variants: [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: bundleArn(input.control.configBundle), + bundleVersion: input.control.bundleVersion, + }, + }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { + configurationBundle: { + bundleArn: bundleArn(input.treatment.configBundle), + bundleVersion: input.treatment.bundleVersion, + }, + }, + }, + ], + evaluationConfig: { + onlineEvaluationConfigArn: `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${input.onlineEval}`, + }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: randomUUID(), + }; + }, + options, + ); + } + + async createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise { + const treatmentWeight = input.treatmentWeight ?? 50; + return this.createABTest( + input.name, + input.gateway, + input.roleArn, + ({ gatewayArn, accountId, roleArn }) => { + const evalArn = (id: string) => + `arn:aws:bedrock-agentcore:${options.region}:${accountId}:online-evaluation-config/${id}`; + return { + name: input.name, + gatewayArn, + variants: [ + { + name: "C", + weight: 100 - treatmentWeight, + variantConfiguration: { target: { name: input.control.gatewayTarget } }, + }, + { + name: "T1", + weight: treatmentWeight, + variantConfiguration: { target: { name: input.treatment.gatewayTarget } }, + }, + ], + evaluationConfig: { + perVariantOnlineEvaluationConfig: [ + { name: "C", onlineEvaluationConfigArn: evalArn(input.control.onlineEval) }, + { name: "T1", onlineEvaluationConfigArn: evalArn(input.treatment.onlineEval) }, + ], + }, + roleArn, + gatewayFilter: input.gatewayFilter, + enableOnCreate: input.enableOnCreate ?? true, + clientToken: randomUUID(), + }; + }, + options, + ); + } + async listBatchInsights( nextToken: string | undefined, maxResults: number | undefined, diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.dfbf3b533e35cebb.json b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.dfbf3b533e35cebb.json new file mode 100644 index 000000000..45ea7f56c --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateABTestCommand.dfbf3b533e35cebb.json @@ -0,0 +1,10 @@ +{ + "abTestId": "agentcore_cli_abtest_tb_run-a0a822a9f4", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_tb_run-a0a822a9f4", + "status": "CREATING", + "executionStatus": "NOT_STARTED", + "createdAt": { + "$date": "2026-08-31T19:41:00.464Z" + }, + "name": "agentcore_cli_abtest_tb_run" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.5a673b1c97a978e3.json b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.5a673b1c97a978e3.json new file mode 100644 index 000000000..62a047c33 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.5a673b1c97a978e3.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_tb_eval_c-ZjE12UEs99", + "onlineEvaluationConfigId": "agentcore_cli_abtest_tb_eval_c-ZjE12UEs99", + "createdAt": { + "$date": "2026-08-31T19:40:25.319Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_tb_eval_c-ZjE12UEs99" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.d708b088d9fe710c.json b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.d708b088d9fe710c.json new file mode 100644 index 000000000..cf1515f28 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateOnlineEvaluationConfigCommand.d708b088d9fe710c.json @@ -0,0 +1,14 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_tb_eval_t1-1eWDkQ4gBO", + "onlineEvaluationConfigId": "agentcore_cli_abtest_tb_eval_t1-1eWDkQ4gBO", + "createdAt": { + "$date": "2026-08-31T19:40:43.703Z" + }, + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_tb_eval_t1-1eWDkQ4gBO" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.22545586b4e16c47.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.22545586b4e16c47.json new file mode 100644 index 000000000..7deaa5628 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.22545586b4e16c47.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreABTest-agentcore_cli_abtest_tb_run-16f47b77", + "RoleId": "AROAZ7CHXJWH4HWBORZGR", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreABTest-agentcore_cli_abtest_tb_run-16f47b77", + "CreateDate": { + "$date": "2026-08-31T19:40:43.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%2C%22Condition%22%3A%7B%22StringEquals%22%3A%7B%22aws%3ASourceAccount%22%3A%22685197708687%22%7D%2C%22ArnLike%22%3A%7B%22aws%3ASourceArn%22%3A%22arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A685197708687%3Aab-test%2F%2A%22%7D%7D%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.2aeccd8e80276a60.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.2aeccd8e80276a60.json new file mode 100644 index 000000000..5bb32cf40 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.2aeccd8e80276a60.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_tb_eval_c", + "RoleId": "AROAZ7CHXJWH4UU6YPSPM", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_tb_eval_c", + "CreateDate": { + "$date": "2026-08-31T19:40:10.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.9a5ae5709d0154a4.json b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.9a5ae5709d0154a4.json new file mode 100644 index 000000000..01b666438 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/CreateRoleCommand.9a5ae5709d0154a4.json @@ -0,0 +1,12 @@ +{ + "Role": { + "Path": "/", + "RoleName": "AgentCoreOnlineEval-agentcore_cli_abtest_tb_eval_t1", + "RoleId": "AROAZ7CHXJWHRPXCVBAFD", + "Arn": "arn:aws:iam::685197708687:role/AgentCoreOnlineEval-agentcore_cli_abtest_tb_eval_t1", + "CreateDate": { + "$date": "2026-08-31T19:40:28.000Z" + }, + "AssumeRolePolicyDocument": "%7B%22Version%22%3A%222012-10-17%22%2C%22Statement%22%3A%5B%7B%22Effect%22%3A%22Allow%22%2C%22Principal%22%3A%7B%22Service%22%3A%22bedrock-agentcore.amazonaws.com%22%7D%2C%22Action%22%3A%22sts%3AAssumeRole%22%7D%5D%7D" + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.75d04f026d22a5c0.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.75d04f026d22a5c0.json new file mode 100644 index 000000000..6e054518f --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.75d04f026d22a5c0.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "NoSuchEntityException", + "message": "The role with name AgentCoreABTest-agentcore_cli_abtest_tb_run-16f47b77 cannot be found." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.9d8a3d9044652466.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.9d8a3d9044652466.json new file mode 100644 index 000000000..fda8f477d --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.9d8a3d9044652466.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "NoSuchEntityException", + "message": "The role with name AgentCoreOnlineEval-agentcore_cli_abtest_tb_eval_t1 cannot be found." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c0088ded17b17b87.json b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c0088ded17b17b87.json new file mode 100644 index 000000000..4f3b2e62d --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/GetRoleCommand.c0088ded17b17b87.json @@ -0,0 +1,6 @@ +{ + "$error": { + "name": "NoSuchEntityException", + "message": "The role with name AgentCoreOnlineEval-agentcore_cli_abtest_tb_eval_c cannot be found." + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.446206df31252f99.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.446206df31252f99.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.446206df31252f99.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.5263c2ec6602ec51.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.5263c2ec6602ec51.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.5263c2ec6602ec51.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.b75418e24aa092ce.json b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.b75418e24aa092ce.json new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/PutRolePolicyCommand.b75418e24aa092ce.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/tb-online-eval-c.golden.json b/src/handlers/eval/ab-test/__fixtures__/tb-online-eval-c.golden.json new file mode 100644 index 000000000..5fb7d9619 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/tb-online-eval-c.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_tb_eval_c-ZjE12UEs99", + "onlineEvaluationConfigId": "agentcore_cli_abtest_tb_eval_c-ZjE12UEs99", + "createdAt": "2026-08-31T19:40:25.319Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_tb_eval_c-ZjE12UEs99" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/tb-online-eval-t1.golden.json b/src/handlers/eval/ab-test/__fixtures__/tb-online-eval-t1.golden.json new file mode 100644 index 000000000..45b9027de --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/tb-online-eval-t1.golden.json @@ -0,0 +1,12 @@ +{ + "onlineEvaluationConfigArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:online-evaluation-config/agentcore_cli_abtest_tb_eval_t1-1eWDkQ4gBO", + "onlineEvaluationConfigId": "agentcore_cli_abtest_tb_eval_t1-1eWDkQ4gBO", + "createdAt": "2026-08-31T19:40:43.703Z", + "status": "CREATING", + "executionStatus": "DISABLED", + "outputConfig": { + "cloudWatchConfig": { + "logGroupName": "/aws/bedrock-agentcore/evaluations/results/agentcore_cli_abtest_tb_eval_t1-1eWDkQ4gBO" + } + } +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/__fixtures__/tb-run.golden.json b/src/handlers/eval/ab-test/__fixtures__/tb-run.golden.json new file mode 100644 index 000000000..2543047a0 --- /dev/null +++ b/src/handlers/eval/ab-test/__fixtures__/tb-run.golden.json @@ -0,0 +1,8 @@ +{ + "abTestId": "agentcore_cli_abtest_tb_run-a0a822a9f4", + "abTestArn": "arn:aws:bedrock-agentcore:us-west-2:685197708687:ab-test/agentcore_cli_abtest_tb_run-a0a822a9f4", + "status": "CREATING", + "executionStatus": "NOT_STARTED", + "createdAt": "2026-08-31T19:41:00.464Z", + "name": "agentcore_cli_abtest_tb_run" +} \ No newline at end of file diff --git a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx index a1512d09f..b5429f5d9 100644 --- a/src/handlers/eval/ab-test/ab-test.fixture.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.fixture.test.tsx @@ -28,6 +28,7 @@ const FIXTURES = join(import.meta.dir, "__fixtures__"); // re-record each describe under its own account: // RECORD=1 bun test -t "fixture-backed reads" // RECORD=1 bun test -t "config-based run" +// RECORD=1 bun test -t "target-based run" const FIXTURE_ABTEST_ID = "abvfylatest_abtargettest-a5f5674e07"; const MISSING_ABTEST_ID = "missing-abtest-0000000000"; @@ -40,6 +41,13 @@ const BUNDLE_NAME = "agentcore_cli_abtest_run_bundle"; const ONLINE_EVAL_NAME = "agentcore_cli_abtest_run_eval"; const AB_TEST_NAME = "agentcore_cli_abtest_run"; +const TB_AB_TEST_NAME = "agentcore_cli_abtest_tb_run"; +const TB_ONLINE_EVAL_C = "agentcore_cli_abtest_tb_eval_c"; +const TB_ONLINE_EVAL_T1 = "agentcore_cli_abtest_tb_eval_t1"; +const TB_GATEWAY_ID = "agentcore-cli-gateway-read-fixture-a-l6opkbe2kd"; +const TB_TARGET_C = "agentcore-cli-gateway-read-target-a1"; +const TB_TARGET_T1 = "agentcore-cli-gateway-read-target-a2"; + const COMPONENTS_V1 = { [RUNTIME_ARN]: { configuration: { system_prompt: "A/B run fixture v1." } }, }; @@ -230,3 +238,102 @@ describe("eval ab-test config-based run", () => { expect(abTest.executionStatus).toBe("NOT_STARTED"); }, 180_000); }); + +const createdTB: { evalC?: string; evalT1?: string; abTestId?: string } = {}; + +afterAll(async () => { + if (!isRecording()) return; + const control = createControlClient({ region: REGION }); + const data = createDataClient({ region: REGION }); + if (createdTB.abTestId) { + try { + await data.send(new DeleteABTestCommand({ abTestId: createdTB.abTestId })); + } catch (error) { + console.error("cleanup tb ab-test:", error); + } + try { + await deleteAbTestRole( + createIamClient({ region: REGION }), + abTestExecutionRoleName(TB_AB_TEST_NAME), + ); + } catch (error) { + console.error("cleanup tb ab-test role:", error); + } + } + for (const id of [createdTB.evalC, createdTB.evalT1]) { + if (!id) continue; + try { + await control.send(new DeleteOnlineEvaluationConfigCommand({ onlineEvaluationConfigId: id })); + } catch (error) { + console.error("cleanup tb online-eval:", error); + } + } +}); + +describe("eval ab-test target-based run", () => { + test("provisions two paused online evaluation configs", async () => { + const c = await run([ + "eval", + "online-eval", + "create", + "--name", + TB_ONLINE_EVAL_C, + "--agent", + AGENT_ID, + "--evaluator", + EVALUATOR_ID, + "--sampling-rate", + "100", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "tb-online-eval-c.golden.json", c); + createdTB.evalC = JSON.parse(c).onlineEvaluationConfigId; + + await settle(); + + const t1 = await run([ + "eval", + "online-eval", + "create", + "--name", + TB_ONLINE_EVAL_T1, + "--agent", + AGENT_ID, + "--evaluator", + EVALUATOR_ID, + "--sampling-rate", + "100", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "tb-online-eval-t1.golden.json", t1); + createdTB.evalT1 = JSON.parse(t1).onlineEvaluationConfigId; + }, 180_000); + + test("runs a paused target-based A/B test", async () => { + const out = await run([ + "eval", + "ab-test", + "target-based", + "run", + "--name", + TB_AB_TEST_NAME, + "--gateway", + TB_GATEWAY_ID, + "--control", + JSON.stringify({ "gateway-target": TB_TARGET_C, "online-eval": createdTB.evalC }), + "--treatment", + JSON.stringify({ "gateway-target": TB_TARGET_T1, "online-eval": createdTB.evalT1 }), + "--treatment-weight", + "20", + "--enable-on-create", + "false", + ]); + matchGolden(FIXTURES, "tb-run.golden.json", out); + const abTest = JSON.parse(out); + createdTB.abTestId = abTest.abTestId; + expect(abTest.abTestId).toBeString(); + expect(abTest.executionStatus).toBe("NOT_STARTED"); + }, 180_000); +}); diff --git a/src/handlers/eval/ab-test/ab-test.test.tsx b/src/handlers/eval/ab-test/ab-test.test.tsx index e2a325e86..a885052e1 100644 --- a/src/handlers/eval/ab-test/ab-test.test.tsx +++ b/src/handlers/eval/ab-test/ab-test.test.tsx @@ -79,9 +79,12 @@ describe("eval ab-test command hierarchy", () => { "stop", "delete", "config-based", + "target-based", ]); const cb = group?.children().find((c) => c.name() === "config-based"); expect(cb?.children().map((c) => c.name())).toEqual(["run"]); + const tb = group?.children().find((c) => c.name() === "target-based"); + expect(tb?.children().map((c) => c.name())).toEqual(["run"]); }); }); @@ -264,3 +267,86 @@ describe("eval ab-test config-based run validation", () => { }); }); }); + +describe("eval ab-test target-based run validation", () => { + const TB_BASE = [ + "eval", + "ab-test", + "target-based", + "run", + "--name", + "orders-v2-canary", + "--gateway", + "orders-gateway-abc123", + "--control", + '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}', + "--treatment", + '{"gateway-target":"orders-v2-target","online-eval":"v2-quality"}', + "--json", + ]; + + test.each(["name", "gateway", "control", "treatment"] as const)( + "requires --%s", + async (missing) => { + const args = TB_BASE.filter( + (a, i) => a !== `--${missing}` && TB_BASE[i - 1] !== `--${missing}`, + ); + await expect(run(args)).rejects.toThrow(new RegExp(`--${missing}`)); + }, + ); + + test("rejects a mis-shaped --control object", async () => { + const args = TB_BASE.map((a) => + a === '{"gateway-target":"orders-prod-target","online-eval":"prod-quality"}' + ? '{"wrong":"shape"}' + : a, + ); + await expect(run(args)).rejects.toThrow(/--control must be/); + }); + + test("rejects identical control/treatment targets", async () => { + const same = '{"gateway-target":"t","online-eval":"e"}'; + await expect( + run([ + "eval", + "ab-test", + "target-based", + "run", + "--name", + "x", + "--gateway", + "g", + "--control", + same, + "--treatment", + same, + "--json", + ]), + ).rejects.toThrow(/different gateway targets/); + }); + + test("maps flags to a createTargetBasedABTest call", async () => { + const { core } = await run([...TB_BASE, "--treatment-weight", "20"], (c) => + c.eval.setAbTestCreateResponse({ + abTestId: "x", + abTestArn: ARN, + name: "x", + status: "CREATING", + executionStatus: "RUNNING", + createdAt: new Date("2026-08-26T10:00:00.000Z"), + }), + ); + const call = core.eval.calls.find((c) => c.method === "createTargetBasedABTest"); + expect(call).toBeDefined(); + expect(call!.args[0]).toEqual({ + name: "orders-v2-canary", + gateway: "orders-gateway-abc123", + control: { gatewayTarget: "orders-prod-target", onlineEval: "prod-quality" }, + treatment: { gatewayTarget: "orders-v2-target", onlineEval: "v2-quality" }, + treatmentWeight: 20, + gatewayFilter: undefined, + roleArn: undefined, + enableOnCreate: undefined, + }); + }); +}); diff --git a/src/handlers/eval/ab-test/index.tsx b/src/handlers/eval/ab-test/index.tsx index e47d42c09..243f77aa3 100644 --- a/src/handlers/eval/ab-test/index.tsx +++ b/src/handlers/eval/ab-test/index.tsx @@ -10,6 +10,7 @@ import { createResumeAbTestHandler } from "./resume"; import { createStopAbTestHandler } from "./stop"; import { createDeleteAbTestHandler } from "./delete"; import { createConfigBasedAbTestHandler } from "./config-based"; +import { createTargetBasedAbTestHandler } from "./target-based"; export function createAbTestHandler(core: Core, io: AppIO): Router { return new Router("ab-test", "inspect AgentCore A/B tests") @@ -22,7 +23,8 @@ export function createAbTestHandler(core: Core, io: AppIO): Router { .handler(createResumeAbTestHandler(core)) .handler(createStopAbTestHandler(core)) .handler(createDeleteAbTestHandler(core)) - .handler(createConfigBasedAbTestHandler(core, io)); + .handler(createConfigBasedAbTestHandler(core, io)) + .handler(createTargetBasedAbTestHandler(core, io)); } export { AbTestScreen } from "./screen.tsx"; diff --git a/src/handlers/eval/ab-test/target-based/index.tsx b/src/handlers/eval/ab-test/target-based/index.tsx new file mode 100644 index 000000000..e13f20b18 --- /dev/null +++ b/src/handlers/eval/ab-test/target-based/index.tsx @@ -0,0 +1,10 @@ +import { Router } from "../../../../router"; +import type { AppIO } from "../../../../io"; +import type { Core } from "../../../types"; +import { createTargetBasedRunHandler } from "./run"; + +export function createTargetBasedAbTestHandler(core: Core, io: AppIO): Router { + return new Router("target-based", "target-based A/B tests").handler( + createTargetBasedRunHandler(core, io), + ); +} diff --git a/src/handlers/eval/ab-test/target-based/run/index.tsx b/src/handlers/eval/ab-test/target-based/run/index.tsx new file mode 100644 index 000000000..609bebc95 --- /dev/null +++ b/src/handlers/eval/ab-test/target-based/run/index.tsx @@ -0,0 +1,116 @@ +import z from "zod"; +import { createHandler, flag } from "../../../../../router"; +import { InputValidationError } from "../../../../../errors"; +import { JsonRendererKey } from "../../../../../tui"; +import { SourceResolver, type AppIO } from "../../../../../io"; +import type { Core } from "../../../../types"; +import type { TargetVariantRef } from "../../../types"; +import { coreOptsFromCtx, parseJsonFlag } from "../../../../utils"; + +const targetRefSchema = z + .object({ + "gateway-target": z.string().min(1), + "online-eval": z.string().min(1), + }) + .strict(); + +function toTargetRef(name: string, raw: unknown): TargetVariantRef { + const parsed = targetRefSchema.safeParse(raw); + if (!parsed.success) { + throw new InputValidationError( + `--${name} must be {"gateway-target": "", "online-eval": ""}`, + ); + } + return { gatewayTarget: parsed.data["gateway-target"], onlineEval: parsed.data["online-eval"] }; +} + +export const createTargetBasedRunHandler = (core: Core, io: AppIO) => + createHandler({ + name: "run", + description: "run an A/B test between two gateway targets and their online evaluations", + flags: [ + flag("name", "the A/B test name", z.string().optional()), + flag("gateway", "deployed gateway id", z.string().optional()), + flag( + "control", + 'control JSON {"gateway-target":"","online-eval":""} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment", + 'treatment JSON {"gateway-target":"","online-eval":""} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "treatment-weight", + "1-99; control weight = 100 - this (default 50)", + z.number().int().optional(), + ), + flag( + "gateway-filter", + 'GatewayFilter JSON, e.g. {"targetPaths":["/orders"]} (inline, file://, or -)', + z.string().optional(), + ), + flag( + "role-arn", + "execution-role override (default: auto-provisioned)", + z.string().optional(), + ), + flag( + "enable-on-create", + "whether to start the test immediately (default true; pass false to create it paused)", + z.enum(["true", "false"]).optional(), + ), + ], + handle: async (ctx, flags) => { + const required = ["name", "gateway", "control", "treatment"] as const; + for (const f of required) { + if (!flags[f]) throw new InputValidationError(`required option '--${f}' not specified`); + } + + const source = new SourceResolver({ stdin: io.stdin }); + const controlRaw = parseJsonFlag( + "control", + await source.resolveText("control", flags["control"]), + ); + const treatmentRaw = parseJsonFlag( + "treatment", + await source.resolveText("treatment", flags["treatment"]), + ); + const gatewayFilter = parseJsonFlag< + import("@aws-sdk/client-bedrock-agentcore").GatewayFilter + >("gateway-filter", await source.resolveText("gateway-filter", flags["gateway-filter"])); + + const control = toTargetRef("control", controlRaw); + const treatment = toTargetRef("treatment", treatmentRaw); + if (control.gatewayTarget === treatment.gatewayTarget) { + throw new InputValidationError( + "control and treatment must reference different gateway targets", + ); + } + + const treatmentWeight = flags["treatment-weight"]; + if (treatmentWeight !== undefined && (treatmentWeight < 1 || treatmentWeight > 99)) { + throw new InputValidationError("--treatment-weight must be between 1 and 99"); + } + + const result = await core.eval.createTargetBasedABTest( + { + name: flags["name"]!, + gateway: flags["gateway"]!, + control, + treatment, + treatmentWeight, + gatewayFilter, + roleArn: flags["role-arn"], + enableOnCreate: + flags["enable-on-create"] === undefined + ? undefined + : flags["enable-on-create"] === "true", + }, + coreOptsFromCtx(ctx), + ); + + ctx.require(JsonRendererKey).renderJson(result); + }, + }); diff --git a/src/handlers/eval/types.tsx b/src/handlers/eval/types.tsx index 4fd9a91ca..8d84c3656 100644 --- a/src/handlers/eval/types.tsx +++ b/src/handlers/eval/types.tsx @@ -247,6 +247,19 @@ export type CreateConfigBasedABTestInput = { enableOnCreate?: boolean; }; +export type TargetVariantRef = { gatewayTarget: string; onlineEval: string }; + +export type CreateTargetBasedABTestInput = { + name: string; + gateway: string; + control: TargetVariantRef; + treatment: TargetVariantRef; + treatmentWeight?: number; + gatewayFilter?: GatewayFilter; + roleArn?: string; + enableOnCreate?: boolean; +}; + export type CreateDatasetInput = CreateDatasetRequest; export type StartRecommendationInput = { name: string; @@ -447,6 +460,10 @@ export interface CoreEvalClient { input: CreateConfigBasedABTestInput, options: CoreOptions, ): Promise; + createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise; // startBatchEvaluation submits an async, service-side evaluation over sessions // the service gathers from the resolved data source. Returns the durable job id // + RUNNING status; poll with getBatchEvaluation. diff --git a/src/testing/TestCoreClient.tsx b/src/testing/TestCoreClient.tsx index 35cbb16d1..6641c7159 100644 --- a/src/testing/TestCoreClient.tsx +++ b/src/testing/TestCoreClient.tsx @@ -148,6 +148,7 @@ import type { CoreEvalClient, CreateConfigurationBundleInput, CreateConfigBasedABTestInput, + CreateTargetBasedABTestInput, CreateDatasetInput, CreateOnlineEvalInput, CreateOnlineInsightInput, @@ -1927,6 +1928,15 @@ export class TestEvalClient implements CoreEvalClient { return this.abTestCreateResponse; } + async createTargetBasedABTest( + input: CreateTargetBasedABTestInput, + options: CoreOptions, + ): Promise { + this.calls.push({ method: "createTargetBasedABTest", args: [input, options] }); + if (this.error) throw this.error; + return this.abTestCreateResponse; + } + async startBatchEvaluation( input: StartBatchEvaluationInput, options: CoreOptions,