diff --git a/packages/client/README.md b/packages/client/README.md index c3895102..924aefdf 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -52,7 +52,7 @@ No code changes are required — `init_client()` detects the packages at runtime The evaluations harness reads an LD-hosted dataset, creates a new evaluation and API-source run, and invokes your handler once per row. Rows can then be scored by LaunchDarkly judges and local scorer functions; see [Score rows with judges and scorers](#score-rows-with-judges-and-scorers). Each success or error queues a `$ld:ai:offline-evals:generation` custom event containing the evaluation, run, dataset, and row identifiers plus output or error (`errorMessage` is included for `ERROR` rows), nested `usage.inputTokens`/`usage.outputTokens`, timing, and stable hashes. Dataset-owned input, expected output, metadata, and variables are not duplicated in the event. Each queued event is logged at `INFO` on the `launchdarkly_ai_server.evaluations.runner` logger with its RFC3339 UTC `emittedAt` timestamp and stable `eventId`, making it possible to compare SDK emission time with ClickHouse arrival time once that logger is enabled. The same `emittedAt` value is included in the event payload. Events are flushed before the summary is fetched and the call returns; handlers are never rerun to retry event delivery. Pass/fail is derived from LaunchDarkly's run summary. -Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A run passes only when the completed summary has no failed, error, or pending rows. `failed_rows` counts rows whose criteria were scored and did not meet their threshold, so a gate that ignored it would exit 0 on a run where every row failed its judge. Evaluation keys must be unique because every call creates a new evaluation with `POST`. +Management API requests go to the `api_base_uri` option, then `LD_API_BASE_URI`, then `https://app.launchdarkly.com`; `LD_BASE_URI` is never used, because it may point at a relay proxy. Result links use `ui_base_uri`, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; this is independent of `LD_API_BASE_URI`. At either level a blank value counts as unset, and a trailing `/` is stripped. After flushing generation events, the harness polls the run summary endpoint until passed + failed + error rows fully account for a nonzero total with no pending rows, polling every `poll_interval_seconds` (default 2s) up to `poll_timeout_seconds` (default 180s); pass either to `run()` to widen both for large datasets. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. A run passes only when the completed summary has no failed, error, or pending rows. `failed_rows` counts rows whose criteria were scored and did not meet their threshold, so a gate that ignored it would exit 0 on a run where every row failed its judge. Evaluation keys must be unique because every call creates a new evaluation with `POST`. ```python import asyncio diff --git a/packages/client/agents.md b/packages/client/agents.md index 5b3a93f9..b93877a7 100644 --- a/packages/client/agents.md +++ b/packages/client/agents.md @@ -164,7 +164,7 @@ Handlers may return any of these — the client normalizes them before emitting Evaluations from Code is experimental. Its public names are exported from `launchdarkly_ai_server.experimental.evaluations` and pinned by `TestPackageExports` in `tests/test_evaluations.py`; none may be added to the package root. `init_evaluations` reports the helper `experimental.initEvaluations`. -`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host `LD_API_BASE_URI`. Do not reuse `LD_BASE_URI`: that variable configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. An event transport is resolved in `init_evaluations()`, which raises before any network I/O when it finds neither an SDK key (`sdk_key` or `LD_SDK_KEY`) nor an already-initialized event-capable client: generation events are the only ingest path for row results, so a run without a transport could never complete. The lifecycle module's bring-your-own-client path (`init_client(client=...)`) therefore satisfies the check on its own, and `run()` reuses that singleton through `_resolve_client`; `run()` raises if the client disappears before it emits. Both polling arguments reject NaN, which would otherwise never compare past a deadline and hang the run. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. +`init_evaluations()` creates an evaluations harness using `LD_API_TOKEN` and the management API host (`api_base_uri`, then `LD_API_BASE_URI`; a blank value at either level counts as unset). Do not reuse `LD_BASE_URI`, or name the option `base_uri`: that host configures SDK delivery and may point at a relay proxy. Evaluation-run links use the separate `ui_base_uri` option, then `LD_UI_BASE_URI`, then `https://app.launchdarkly.com`; do not derive their host from `LD_API_BASE_URI`. An event transport is resolved in `init_evaluations()`, which raises before any network I/O when it finds neither an SDK key (`sdk_key` or `LD_SDK_KEY`) nor an already-initialized event-capable client: generation events are the only ingest path for row results, so a run without a transport could never complete. The lifecycle module's bring-your-own-client path (`init_client(client=...)`) therefore satisfies the check on its own, and `run()` reuses that singleton through `_resolve_client`; `run()` raises if the client disappears before it emits. Both polling arguments reject NaN, which would otherwise never compare past a deadline and hang the run. The harness always queues one `$ld:ai:offline-evals:generation` custom event per row through the standard SDK event transport and flushes before returning. No feature flag gates event emission. The harness polls the run summary endpoint until a nonzero `total_rows` has `pending_rows == 0` and `passed + failed + error` rows accounting for the total, polling every `poll_interval_seconds` (default 2s) until `poll_timeout_seconds` (default 180s); both are `run()` arguments so large datasets can widen them. The summary endpoint does not return run state, so `RunSummary` exposes row counts only. `init_evaluations()` takes `project_key`; `run()` does not. Dataset lookup/row pagination, evaluation creation, and run creation are private helpers; `run()` and `tools.get()` are the public surface. `tools` is a `list[EvalTool]`. `await evals.tools.get(key, implementation=...)` issues `GET projects/

/ai-tools/`, pins the returned version, and builds a library tool through `EvalTool._library`. `get` is a coroutine that reads in a worker thread. `EvalTool` is frozen with `schema` required, so a constructed tool is always inline. `_validate_tools` also rejects a library tool that has no `project_key` or one from another project. `run()` issues no tool request at all: a library tool was already read by `tools.get()`. Library tools are sent as `{key, version, source: "library"}` and inline tools as `{key, schema, description, source: "inline"}`. `_validate_tools` runs in `run()` before any I/O, so a bad list — a non-`EvalTool` entry, a blank or uppercase key, a non-object or non-serializable `schema` (`allow_nan=False`), a non-callable implementation, a repeated key (compared without case), or a `NativeTool` on an inline tool — fails with zero requests recorded. `_validate_tool_key` is shared by `tools.get()` and `_validate_tools`, so the key rules are identical on both paths. Handler config synthesis is unforked: `config["tools"][tool.key] = {"description", "parameters"}` is fed from whichever kind the tool is, so handlers cannot tell them apart, and `_tool_handlers` maps each tool to its executable. Each call creates a new evaluation with `POST` and a run with `source="api"`, so its key must be unique. `run()` takes `handler` or `handlers`, never both, and every handler must declare `provides_for`. The generation config and each judge select a handler with `utils.select_handler`, the `config()` rule, with no mode fallback. The prompt must match the handler mode (`prompt_mode_error`), and judges get no tools. With `ai_config`, `_read_ai_config` reads the AI Config mode, the variation, and the model config in that order, and runs each check right after the read that supplies its input. The harness directly invokes the selected handler once per row and never retries it — event delivery is never a reason to rerun a handler because that would repeat tool side effects; retries apply only to management API requests. A 429 is replayed for any method, but 5xx responses and transport failures are replayed only for `GET`/`HEAD`, so an evaluation or run `POST` that the server may already have applied is never duplicated. Management API calls run in a worker thread (`asyncio.to_thread`) because the client is synchronous; the caller's event loop stays free. Generation events go through the already-initialized SDK client when the application has one — `init_client` is idempotent, so an existing singleton wins and the evaluations SDK key is ignored with a warning. Dataset-owned `input`, `expected_output`, `metadata`, and `variables` are deliberately excluded from the event payload. The harness flushes events, polls the run summary endpoint until row accounting is complete (`total_rows > 0`, `pending_rows == 0`, and `passed + failed + error == total_rows`), and raises a timeout once `poll_timeout_seconds` elapses if the backend never reaches one. `RunSummary` includes row counts only, and `EvalRunResult.passed` is true only when error and pending row counts are both zero. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/module.py b/packages/client/src/launchdarkly_ai_server/evaluations/module.py index f53d7f52..61f3919e 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/module.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/module.py @@ -152,6 +152,11 @@ def _env(name: str) -> str | None: return value if value else None +def _resolve_host(option: str | None, env_name: str, default: str) -> str: + """Resolve a host: option, then env var, then default. Blank counts as unset.""" + return (option or "").strip() or _env(env_name) or default + + def _initialized_client() -> Any | None: """Return the SDK singleton when one is initialized, otherwise ``None``.""" try: @@ -866,7 +871,7 @@ def init_evaluations( project_key: str | None = None, api_key: str | None = None, sdk_key: str | None = None, - base_uri: str | None = None, + api_base_uri: str | None = None, ui_base_uri: str | None = None, transport: Transport = urllib_transport, ) -> EvaluationsModule: @@ -907,12 +912,12 @@ def init_evaluations( api_client = LDApiClient( api_key=token, - base_uri=base_uri or _env("LD_API_BASE_URI") or DEFAULT_BASE_URI, + base_uri=_resolve_host(api_base_uri, "LD_API_BASE_URI", DEFAULT_BASE_URI), transport=transport, ) return EvaluationsModule( api_client=api_client, project_key=resolved_project_key, sdk_key=resolved_sdk_key, - ui_base_uri=ui_base_uri or _env("LD_UI_BASE_URI") or DEFAULT_UI_BASE_URI, + ui_base_uri=_resolve_host(ui_base_uri, "LD_UI_BASE_URI", DEFAULT_UI_BASE_URI), ) diff --git a/packages/client/tests/test_evaluations.py b/packages/client/tests/test_evaluations.py index f20edf2e..828a19ef 100644 --- a/packages/client/tests/test_evaluations.py +++ b/packages/client/tests/test_evaluations.py @@ -167,41 +167,148 @@ def test_missing_sdk_key_raises_when_the_byoc_client_cannot_emit_events( init_evaluations(transport=failing_transport) -def test_base_uri_override_isolated_from_sdk_delivery_uri( - monkeypatch: pytest.MonkeyPatch, -) -> None: +@pytest.fixture +def host_env(monkeypatch: pytest.MonkeyPatch) -> pytest.MonkeyPatch: monkeypatch.setenv("LD_PROJECT_KEY", "proj") monkeypatch.setenv("LD_API_TOKEN", "api-token") monkeypatch.setenv("LD_SDK_KEY", "sdk-key") - monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com/") - monkeypatch.setenv("LD_BASE_URI", "https://relay.example.com/") + for name in ("LD_API_BASE_URI", "LD_UI_BASE_URI", "LD_BASE_URI"): + monkeypatch.delenv(name, raising=False) + return monkeypatch + + +def request_url(**options: Any) -> str: + """Init the harness, send one management API request, return its URL.""" + transport = RecordingTransport() + evals = init_evaluations(transport=transport, **options) + evals.api.get("projects/proj/evaluations") + return str(transport.requests[0]["url"]) + + +def test_api_host_option_wins_over_env(host_env: pytest.MonkeyPatch) -> None: + host_env.setenv("LD_API_BASE_URI", "https://api.staging.example.com") + + url = request_url(api_base_uri="https://other.example.com") + + assert url == "https://other.example.com/api/v2/projects/proj/evaluations" - from_env = init_evaluations(transport=RecordingTransport()) - explicit = init_evaluations( - base_uri="https://other.example.com", transport=RecordingTransport() + +def test_api_host_env_wins_over_default(host_env: pytest.MonkeyPatch) -> None: + host_env.setenv("LD_API_BASE_URI", "https://api.staging.example.com") + + url = request_url() + + assert url == "https://api.staging.example.com/api/v2/projects/proj/evaluations" + + +def test_api_host_defaults_to_production(host_env: pytest.MonkeyPatch) -> None: + assert ( + request_url() == "https://app.launchdarkly.com/api/v2/projects/proj/evaluations" ) - assert from_env.api.base_uri == "https://api.staging.example.com" - assert explicit.api.base_uri == "https://other.example.com" +@pytest.mark.parametrize("blank", ["", " "]) +def test_blank_api_host_option_falls_through_to_env( + host_env: pytest.MonkeyPatch, blank: str +) -> None: + host_env.setenv("LD_API_BASE_URI", "https://api.staging.example.com") -def test_ui_base_uri_precedence_and_api_base_isolation( - monkeypatch: pytest.MonkeyPatch, + url = request_url(api_base_uri=blank) + + assert url == "https://api.staging.example.com/api/v2/projects/proj/evaluations" + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_blank_api_host_at_both_levels_falls_through_to_default( + host_env: pytest.MonkeyPatch, blank: str ) -> None: - monkeypatch.setenv("LD_PROJECT_KEY", "proj") - monkeypatch.setenv("LD_API_TOKEN", "api-token") - monkeypatch.setenv("LD_SDK_KEY", "sdk-key") - monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com") - monkeypatch.setenv("LD_UI_BASE_URI", "https://ui.staging.example.com/") + host_env.setenv("LD_API_BASE_URI", blank) + + url = request_url(api_base_uri=blank) + + assert url == "https://app.launchdarkly.com/api/v2/projects/proj/evaluations" + + +@pytest.mark.parametrize( + "options, env", + [ + ({"api_base_uri": "https://host/"}, None), + ({}, "https://host/"), + ], +) +def test_api_host_trailing_slash_is_stripped( + host_env: pytest.MonkeyPatch, options: dict[str, str], env: str | None +) -> None: + if env is not None: + host_env.setenv("LD_API_BASE_URI", env) + + assert request_url(**options) == "https://host/api/v2/projects/proj/evaluations" + + +def test_sdk_delivery_host_alone_does_not_move_api_requests( + host_env: pytest.MonkeyPatch, +) -> None: + host_env.setenv("LD_BASE_URI", "https://relay.example.com") + + url = request_url() + + assert url == "https://app.launchdarkly.com/api/v2/projects/proj/evaluations" + - from_env = init_evaluations(transport=RecordingTransport()) - explicit = init_evaluations( +def test_init_evaluations_has_no_base_uri_option(host_env: pytest.MonkeyPatch) -> None: + with pytest.raises(TypeError): + init_evaluations(base_uri="https://relay.example.com") # type: ignore[call-arg] + + +def test_ui_host_option_wins_over_env(host_env: pytest.MonkeyPatch) -> None: + host_env.setenv("LD_UI_BASE_URI", "https://ui.staging.example.com") + + evals = init_evaluations( ui_base_uri="https://ui.example.com/", transport=RecordingTransport() ) - assert from_env.api.base_uri == "https://api.staging.example.com" - assert from_env.ui_base_uri == "https://ui.staging.example.com" - assert explicit.ui_base_uri == "https://ui.example.com" + assert evals.ui_base_uri == "https://ui.example.com" + + +def test_ui_host_env_wins_over_default(host_env: pytest.MonkeyPatch) -> None: + host_env.setenv("LD_UI_BASE_URI", "https://ui.staging.example.com/") + + evals = init_evaluations(transport=RecordingTransport()) + + assert evals.ui_base_uri == "https://ui.staging.example.com" + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_blank_ui_host_option_falls_through_to_env( + host_env: pytest.MonkeyPatch, blank: str +) -> None: + host_env.setenv("LD_UI_BASE_URI", "https://ui.staging.example.com") + + evals = init_evaluations(ui_base_uri=blank, transport=RecordingTransport()) + + assert evals.ui_base_uri == "https://ui.staging.example.com" + + +@pytest.mark.parametrize("blank", ["", " "]) +def test_blank_ui_host_at_both_levels_falls_through_to_default( + host_env: pytest.MonkeyPatch, blank: str +) -> None: + host_env.setenv("LD_UI_BASE_URI", blank) + + evals = init_evaluations(ui_base_uri=blank, transport=RecordingTransport()) + + assert evals.ui_base_uri == "https://app.launchdarkly.com" + + +def test_api_host_env_alone_leaves_ui_host_on_default( + host_env: pytest.MonkeyPatch, +) -> None: + host_env.setenv("LD_API_BASE_URI", "https://api.staging.example.com") + + evals = init_evaluations(transport=RecordingTransport()) + + assert evals.api.base_uri == "https://api.staging.example.com" + assert evals.ui_base_uri == "https://app.launchdarkly.com" def test_requests_carry_token_auth_and_json_body() -> None: diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 6cbd189b..d2b5919d 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -254,7 +254,7 @@ async def test_complete_run_with_zero_failed_and_error_rows_passes( project_key="proj", api_key="token", sdk_key="sdk-key", - base_uri="https://api.example.com", + api_base_uri="https://api.example.com", ui_base_uri="https://ui.example.com/", transport=transport, ) @@ -988,6 +988,39 @@ def hosted_dataset_responses() -> list[HttpResponse]: ] +@pytest.mark.asyncio +async def test_api_host_env_alone_leaves_result_url_on_default_ui_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The UI host is not derived from the API host: environments do not always + # serve both from the same place. + monkeypatch.setenv("LD_API_BASE_URI", "https://api.staging.example.com") + monkeypatch.delenv("LD_UI_BASE_URI", raising=False) + transport = SequencedTransport(hosted_dataset_responses()) + evals = init_evaluations( + project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport + ) + + async def handler(*args: object) -> dict[str, Any]: + return {"output": "ok"} + + result = await evals.run( + key="eval-key", + dataset="golden", + handler=tagged(handler), + generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"}, + ) + + assert all( + request["url"].startswith("https://api.staging.example.com/api/v2/") + for request in transport.requests + ) + assert result.url == ( + "https://app.launchdarkly.com/projects/proj/ai/evaluations/" + "evaluation-id/runs/run-id" + ) + + @pytest.mark.asyncio async def test_inline_tool_runs_without_reading_the_tool_api() -> None: transport = SequencedTransport(hosted_dataset_responses())