Skip to content

feat: GenAI Client(evals) - async auto-create EvaluationExperiment parity #7016

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: GenAI Client(evals) - async auto-create EvaluationExperiment parity #7016

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions agentplatform/_genai/_evals_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,21 @@
_DEFAULT_CANDIDATE_NAME = _evals_constant.DEFAULT_CANDIDATE_NAME


def _local_timestamp() -> str:
"""Returns the current local time as 'M/D/YYYY, H:MM:SS AM/PM'.

Matches the Agent Platform UI's default experiment name timestamp format
(e.g. '6/1/2026, 1:12:29 PM').
"""
now = datetime.datetime.now()
hour_12 = now.hour % 12 or 12
meridiem = "AM" if now.hour < 12 else "PM"
return (
f"{now.month}/{now.day}/{now.year}, "
f"{hour_12}:{now.minute:02d}:{now.second:02d} {meridiem}"
)


@contextlib.contextmanager
def _temp_logger_level(logger_name: str, level: int) -> None: # type: ignore[misc]
"""Temporarily sets the level of a logger."""
Expand Down
59 changes: 53 additions & 6 deletions agentplatform/_genai/evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,13 @@ def _CreateEvaluationRunParameters_to_vertex(
[item for item in getv(from_object, ["analysis_configs"])],
)

if getv(from_object, ["evaluation_experiment"]) is not None:
setv(
to_object,
["evaluationExperiment"],
getv(from_object, ["evaluation_experiment"]),
)

return to_object


Expand Down Expand Up @@ -1415,6 +1422,7 @@ def _create_evaluation_run(
] = None,
config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None,
evaluation_experiment: Optional[str] = None,
) -> types.EvaluationRun:
"""
Creates an EvaluationRun.
Expand All @@ -1429,6 +1437,7 @@ def _create_evaluation_run(
inference_configs=inference_configs,
config=config,
analysis_configs=analysis_configs,
evaluation_experiment=evaluation_experiment,
)

request_url_dict: Optional[dict[str, str]]
Expand Down Expand Up @@ -3216,6 +3225,7 @@ def create_evaluation_run(
metrics: list[types.EvaluationRunMetricOrDict],
name: Optional[str] = None,
display_name: Optional[str] = None,
evaluation_experiment: Optional[str] = None,
agent_info: Optional[evals_types.AgentInfoOrDict] = None,
agent: Optional[str] = None,
user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None,
Expand All @@ -3236,6 +3246,11 @@ def create_evaluation_run(
metrics: The list of metrics to evaluate.
name: The name of the evaluation run.
display_name: The display name of the evaluation run.
evaluation_experiment: The resource name of an existing
EvaluationExperiment to group this run under. If omitted, a new
EvaluationExperiment is created automatically so the run is visible in
the Agent Platform UI. Pass an existing experiment name to group
multiple runs together.
agent_info: The agent info to evaluate. Mutually exclusive with
`inference_configs`.
agent: The agent resource name in str type. Accepts either an Agent
Expand Down Expand Up @@ -3394,10 +3409,22 @@ def create_evaluation_run(
self._api_client, resolved_dataset, inference_configs, parsed_agent_info
)
resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent)
resolved_name = name or f"evaluation_run_{uuid.uuid4()}"
resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}"
resolved_experiment = evaluation_experiment
if resolved_experiment is None:
experiment_display_name = (
display_name or f"SDK Experiment {_evals_common._local_timestamp()}"
)
created_experiment = self.create_evaluation_experiment(
display_name=experiment_display_name
)
resolved_experiment = created_experiment.name
# The backend rejects a caller-supplied `name` for runs that belong to an
# EvaluationExperiment and assigns the resource ID itself, so `name` is
# only used as a display_name fallback.
return self._create_evaluation_run(
name=resolved_name,
display_name=display_name or resolved_name,
display_name=resolved_display_name,
evaluation_experiment=resolved_experiment,
data_source=resolved_dataset,
evaluation_config=evaluation_config,
inference_configs=resolved_inference_configs,
Expand Down Expand Up @@ -4039,6 +4066,7 @@ async def _create_evaluation_run(
] = None,
config: Optional[types.CreateEvaluationRunConfigOrDict] = None,
analysis_configs: Optional[list[types.AnalysisConfigOrDict]] = None,
evaluation_experiment: Optional[str] = None,
) -> types.EvaluationRun:
"""
Creates an EvaluationRun.
Expand All @@ -4053,6 +4081,7 @@ async def _create_evaluation_run(
inference_configs=inference_configs,
config=config,
analysis_configs=analysis_configs,
evaluation_experiment=evaluation_experiment,
)

request_url_dict: Optional[dict[str, str]]
Expand Down Expand Up @@ -5475,6 +5504,7 @@ async def create_evaluation_run(
metrics: list[types.EvaluationRunMetricOrDict],
name: Optional[str] = None,
display_name: Optional[str] = None,
evaluation_experiment: Optional[str] = None,
agent_info: Optional[evals_types.AgentInfo] = None,
agent: Optional[str] = None,
user_simulator_config: Optional[evals_types.UserSimulatorConfigOrDict] = None,
Expand All @@ -5495,6 +5525,11 @@ async def create_evaluation_run(
metrics: The list of metrics to evaluate.
name: The name of the evaluation run.
display_name: The display name of the evaluation run.
evaluation_experiment: The resource name of an existing
EvaluationExperiment to group this run under. If omitted, a new
EvaluationExperiment is created automatically so the run is visible in
the Agent Platform UI. Pass an existing experiment name to group
multiple runs together.
agent_info: The agent info to evaluate. Mutually exclusive with
`inference_configs`.
agent: The agent resource name in str type. Accepts either an Agent
Expand Down Expand Up @@ -5653,11 +5688,23 @@ async def create_evaluation_run(
self._api_client, resolved_dataset, inference_configs, parsed_agent_info
)
resolved_labels = _evals_common._add_evaluation_run_labels(labels, agent)
resolved_name = name or f"evaluation_run_{uuid.uuid4()}"
resolved_display_name = display_name or name or f"evaluation_run_{uuid.uuid4()}"
resolved_experiment = evaluation_experiment
if resolved_experiment is None:
experiment_display_name = (
display_name or f"SDK Experiment {_evals_common._local_timestamp()}"
)
created_experiment = await self.create_evaluation_experiment(
display_name=experiment_display_name
)
resolved_experiment = created_experiment.name

# The backend rejects a caller-supplied `name` for runs that belong to an
# EvaluationExperiment and assigns the resource ID itself, so `name` is
# only used as a display_name fallback.
result = await self._create_evaluation_run(
name=resolved_name,
display_name=display_name or resolved_name,
display_name=resolved_display_name,
evaluation_experiment=resolved_experiment,
data_source=resolved_dataset,
evaluation_config=evaluation_config,
inference_configs=resolved_inference_configs,
Expand Down
11 changes: 11 additions & 0 deletions agentplatform/_genai/types/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3017,6 +3017,12 @@ class _CreateEvaluationRunParameters(_common.BaseModel):
analysis_configs: Optional[list[AnalysisConfig]] = Field(
default=None, description=""""""
)
evaluation_experiment: Optional[str] = Field(
default=None,
description="""The resource name of the parent EvaluationExperiment that this run
belongs to. Format:
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`.""",
)


class _CreateEvaluationRunParametersDict(TypedDict, total=False):
Expand Down Expand Up @@ -3046,6 +3052,11 @@ class _CreateEvaluationRunParametersDict(TypedDict, total=False):
analysis_configs: Optional[list[AnalysisConfigDict]]
""""""

evaluation_experiment: Optional[str]
"""The resource name of the parent EvaluationExperiment that this run
belongs to. Format:
`projects/{project}/locations/{location}/evaluationExperiments/{evaluation_experiment}`."""


_CreateEvaluationRunParametersOrDict = Union[
_CreateEvaluationRunParameters, _CreateEvaluationRunParametersDict
Expand Down
14 changes: 10 additions & 4 deletions tests/unit/agentplatform/genai/replays/test_create_evaluation_run.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from tests.unit.agentplatform.genai.replays import pytest_helper
from agentplatform import types
from agentplatform._genai import _evals_common
from google.genai import types as genai_types
import pandas as pd
import pytest
Expand Down Expand Up @@ -106,7 +107,7 @@
CANDIDATE_NAME = "candidate_1"
MODEL_NAME = "projects/977012026409/locations/us-central1/publishers/google/models/gemini-2.5-flash"
EVAL_SET_NAME = (
"projects/977012026409/locations/us-central1/evaluationSets/6619939608513740800"
"projects/977012026409/locations/us-central1/evaluationSets/1936778737211146240"
)


Expand Down Expand Up @@ -319,8 +320,13 @@ def test_create_eval_run_with_allow_cross_region_model(client):
assert evaluation_run.error is None


@mock.patch.object(
_evals_common, "_local_timestamp", return_value="1/1/2026, 12:00:00 AM"
)
@mock.patch("uuid.uuid4")
def test_create_eval_run_with_metric_resource_name(mock_uuid4, client):
def test_create_eval_run_with_metric_resource_name(
mock_uuid4, mock_local_timestamp, client
):
"""Tests create_evaluation_run with metric_resource_name."""
mock_uuid4.side_effect = [
uuid.UUID("d392c573-9e81-4a30-b984-8a6aa4656369"),
Expand Down Expand Up @@ -718,7 +724,7 @@ def test_create_eval_run_with_interactions_data_source(mock_uuid4, client):
"""
mock_uuid4.return_value = uuid.UUID("a1b2c3d4-e5f6-7890-abcd-ef1234567890")
client._api_client._http_options.api_version = "v1beta1"
interaction_id = "ChA2YzllYzk0MjY1NjZjODM5EAgaATAqBG1haW4"
interaction_id = "ChBjYjdiNWIzYTMzMGQ2MjE1EAgaATAqBG1haW4"
gemini_agent = (
"projects/model-evaluation-dev/locations/global/agents/test-agent-eval"
)
Expand Down Expand Up @@ -914,7 +920,7 @@ def test_create_eval_run_with_gemini_agent(client):
)
eval_set = (
"projects/model-evaluation-dev/locations/global/evaluationSets/"
"7392342128979869696"
"4227422097682464768"
)
evaluation_run = client.evals.create_evaluation_run(
name="test_gemini_agent",
Expand Down
Loading
Loading