feat(project): add project add evaluator code-based

src/assets/evaluators/autoevals-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/autoevals-lambda/lambda_function.py

@@ -0,0 +1,37 @@
+{{#if ModelProviderBedrock}}
+import os
+
+# litellm's Bedrock provider reads AWS_REGION_NAME; Lambda only sets AWS_REGION/AWS_DEFAULT_REGION.
+os.environ.setdefault("AWS_REGION_NAME", os.environ.get("AWS_REGION", "us-west-2"))
+
+from autoevals import {{ EvaluatorClass }}, init
+from autoevals.litellm import LiteLLMClient
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+client = LiteLLMClient()
+init(client=client, default_model="bedrock/{{ Model }}")
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}(client=client, model="bedrock/{{ Model }}"){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{else}}
+from autoevals import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.autoevals import AutoEvalsAdapter
+
+adapter = AutoEvalsAdapter(metric={{ EvaluatorClass }}({{#if Model}}model="{{ Model }}"{{/if}}){{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}})
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/autoevals-lambda/pyproject.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (Autoevals)"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore[autoevals]",
+    "autoevals>=0.0.80,<1.0.0",
+{{#if ModelProviderBedrock}}
+    # autoevals grades via LiteLLMClient -> Bedrock (Converse); litellm replaces the openai judge
+    "litellm>=1.60,<1.85",
+{{else}}
+    "openai>=1.0.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/deepeval-lambda/execution-role-policy.json

@@ -0,0 +1,15 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": ["bedrock:InvokeModel"],
+      "Resource": "*"
+    }
+  ]
+}

src/assets/evaluators/deepeval-lambda/lambda_function.py

@@ -0,0 +1,29 @@
+import os
+
+os.environ.setdefault("DEEPEVAL_RESULTS_FOLDER", "/tmp/.deepeval")
+os.environ.setdefault("DEEPEVAL_TELEMETRY_OPT_OUT", "YES")
+os.chdir("/tmp")
+
+{{#if ModelProviderBedrock}}
+from deepeval.models import AmazonBedrockModel
+{{/if}}
+from deepeval.metrics import {{ EvaluatorClass }}
+
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    EvaluatorInput,
+    EvaluatorOutput,
+    custom_code_based_evaluator,
+)
+from bedrock_agentcore.evaluation.custom_code_based_evaluators.third_party.deepeval import DeepEvalAdapter
+
+{{#if ModelProviderBedrock}}
+model = AmazonBedrockModel(model="{{ Model }}", region=os.environ.get("AWS_REGION", "us-west-2"))
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}(model=model{{#if EvaluatorParams}}, {{{ EvaluatorParams }}}{{/if}}))
+{{else}}
+adapter = DeepEvalAdapter(metric={{ EvaluatorClass }}({{{ EvaluatorParams }}}))
+{{/if}}
+
+
+@custom_code_based_evaluator()
+def handler(evaluator_input: EvaluatorInput, context) -> EvaluatorOutput:
+    return adapter(evaluator_input, context)

src/assets/evaluators/deepeval-lambda/pyproject.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator (DeepEval)"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore[deepeval]",
+    "deepeval>=2.0.0,<3.0.0",
+{{#if ModelProviderBedrock}}
+    "aiobotocore>=2.13.0",
+{{/if}}
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/assets/evaluators/python-lambda/execution-role-policy.json

@@ -0,0 +1,10 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"],
+      "Resource": "arn:*:logs:*:*:log-group:/aws/lambda/*"
+    }
+  ]
+}

src/assets/evaluators/python-lambda/lambda_function.py

@@ -0,0 +1,19 @@
+from bedrock_agentcore.evaluation.custom_code_based_evaluators import (
+    custom_code_based_evaluator,
+    EvaluatorInput,
+    EvaluatorOutput,
+)
+
+
+@custom_code_based_evaluator()
+def handler(input: EvaluatorInput, context) -> EvaluatorOutput:
+    """Evaluate agent behavior with custom logic.
+
+    Args:
+        input: Contains evaluation_level, session_spans, target_trace_id, target_span_id
+
+    Returns:
+        EvaluatorOutput with value/label for success, or errorCode/errorMessage for failure.
+    """
+    # TODO: Replace with your evaluation logic
+    return EvaluatorOutput(value=1.0, label="Pass", explanation="Evaluation passed")

src/assets/evaluators/python-lambda/pyproject.toml

@@ -0,0 +1,15 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "{{ Name }}"
+version = "0.1.0"
+description = "AgentCore Code-Based Evaluator"
+requires-python = ">=3.10"
+dependencies = [
+    "bedrock-agentcore>=1.6.0",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["."]

src/core/abTestExecutionRole.test.ts

@@ -0,0 +1,100 @@
+import { test, expect, describe } from "bun:test";
+import { CreateRoleCommand, GetRoleCommand, type IAMClient } from "@aws-sdk/client-iam";
+import {
+  abTestExecutionRoleName,
+  accountIdFromArn,
+  provisionAbTestRole,
+} from "./abTestExecutionRole";
+
+const GATEWAY_ARN = "arn:aws:bedrock-agentcore:us-west-2:123456789012:gateway/orders-gw";
+
+type Sent = { name: string; input: unknown };
+
+function fakeIam(onGet: "found" | "missing"): { iam: IAMClient; sent: Sent[] } {
+  const sent: Sent[] = [];
+  const iam = {
+    send: async (command: { constructor: { name: string }; input: unknown }) => {
+      sent.push({ name: command.constructor.name, input: command.input });
+      if (command instanceof GetRoleCommand) {
+        if (onGet === "missing") {
+          throw Object.assign(new Error("no such entity"), { name: "NoSuchEntityException" });
+        }
+        return {
+          Role: {
+            Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`,
+          },
+        };
+      }
+      if (command instanceof CreateRoleCommand) {
+        return {
+          Role: {
+            Arn: `arn:aws:iam::123456789012:role/${(command.input as { RoleName: string }).RoleName}`,
+          },
+        };
+      }
+      return {};
+    },
+  } as unknown as IAMClient;
+  return { iam, sent };
+}
+
+describe("abTestExecutionRoleName", () => {
+  test("stays within IAM's 64-char limit and is deterministic", () => {
+    const long = abTestExecutionRoleName("x".repeat(120));
+    expect(long.length).toBeLessThanOrEqual(64);
+    expect(abTestExecutionRoleName("orders")).toBe(abTestExecutionRoleName("orders"));
+  });
+
+  test("distinct names for distinct tests", () => {
+    expect(abTestExecutionRoleName("a")).not.toBe(abTestExecutionRoleName("b"));
+  });
+});
+
+describe("accountIdFromArn", () => {
+  test("extracts the account segment", () => {
+    expect(accountIdFromArn(GATEWAY_ARN)).toBe("123456789012");
+  });
+  test("throws on a malformed ARN", () => {
+    expect(() => accountIdFromArn("not-an-arn")).toThrow(/account id/);
+  });
+});
+
+describe("provisionAbTestRole", () => {
+  test("creates the role + inline policy and reports created=true", async () => {
+    const { iam, sent } = fakeIam("missing");
+    const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2");
+
+    expect(result.created).toBe(true);
+    expect(result.roleArn).toContain(":role/");
+    expect(sent.map((s) => s.name)).toEqual([
+      "GetRoleCommand",
+      "CreateRoleCommand",
+      "PutRolePolicyCommand",
+    ]);
+
+    const create = sent.find((s) => s.name === "CreateRoleCommand")!.input as {
+      AssumeRolePolicyDocument: string;
+    };
+    const trust = JSON.parse(create.AssumeRolePolicyDocument);
+    expect(trust.Statement[0].Principal.Service).toBe("bedrock-agentcore.amazonaws.com");
+    expect(trust.Statement[0].Condition.StringEquals["aws:SourceAccount"]).toBe("123456789012");
+    expect(trust.Statement[0].Condition.ArnLike["aws:SourceArn"]).toContain(":ab-test/*");
+
+    const policy = sent.find((s) => s.name === "PutRolePolicyCommand")!.input as {
+      PolicyDocument: string;
+    };
+    const doc = JSON.parse(policy.PolicyDocument);
+    const actions = doc.Statement.flatMap((s: { Action: string[] }) => s.Action);
+    expect(actions).toContain("bedrock-agentcore:GetGateway");
+    expect(actions).toContain("bedrock-agentcore:GetConfigurationBundleVersion");
+    expect(actions).toContain("bedrock-agentcore:GetOnlineEvaluationConfig");
+  });
+
+  test("reuses an existing role and reports created=false", async () => {
+    const { iam, sent } = fakeIam("found");
+    const result = await provisionAbTestRole(iam, "orders-v2", GATEWAY_ARN, "us-west-2");
+
+    expect(result.created).toBe(false);
+    expect(sent.map((s) => s.name)).toEqual(["GetRoleCommand", "PutRolePolicyCommand"]);
+  });
+});