Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 28 additions & 1 deletion apps/server/src/telemetry/ProviderUsageAnalytics.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ const context = (
modelSelection: {
instanceId: ProviderInstanceId.make("codex"),
model,
options: [{ id: "reasoningEffort", value: "xhigh" }],
options: [
{ id: "reasoningEffort", value: "xhigh" },
{ id: "serviceTier", value: "priority" },
],
},
runtimeMode: "full-access",
interactionMode: "default",
Expand Down Expand Up @@ -154,6 +157,7 @@ describe("provider usage reporting", () => {
lifecycleObserved: true,
model: "gpt-6-sol",
effort: "xhigh",
serviceTier: "fast",
runtimeMode: "full-access",
interactionMode: "default",
createdBy: "user",
Expand Down Expand Up @@ -320,4 +324,27 @@ describe("provider usage reporting", () => {
}
expect(routed("ollama/acme-internal", "secret-variant")).not.toHaveProperty("effort");
});

it("reports closed service tiers from a tier option or a fast mode toggle", () => {
const tier = (option: { id: string; value: string | boolean }) =>
providerTurnProperties(
{
driver: "codex",
context: {
modelSelection: {
instanceId: ProviderInstanceId.make("codex"),
model: "gpt-6-sol",
options: [option],
},
},
providerTurn: turn("t", complete(1, 1)),
},
label,
).serviceTier;
expect(tier({ id: "serviceTier", value: "ultrafast" })).toBe("ultrafast");
expect(tier({ id: "serviceTier", value: "default" })).toBe("standard");
expect(tier({ id: "serviceTier", value: "/home/me/secret" })).toBeUndefined();
expect(tier({ id: "fastMode", value: true })).toBe("fast");
expect(tier({ id: "fastMode", value: false })).toBe("standard");
});
});
22 changes: 21 additions & 1 deletion apps/server/src/telemetry/ProviderUsageAnalytics.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@ import * as DateTime from "effect/DateTime";
import * as Effect from "effect/Effect";
import * as Layer from "effect/Layer";

import { getModelSelectionStringOptionValue } from "@t3tools/shared/model";
import {
getModelSelectionBooleanOptionValue,
getModelSelectionStringOptionValue,
} from "@t3tools/shared/model";
import * as ProviderRegistry from "../provider/ProviderRegistry.ts";
import * as AnalyticsService from "./AnalyticsService.ts";
import { catalogModelLabel } from "./ProviderDimensions.ts";
Expand Down Expand Up @@ -106,6 +109,22 @@ function selectedEffort(selection: ModelSelection): string | undefined {
return effort !== undefined && REPORTED_EFFORTS.has(effort) ? effort : undefined;
}

// Codex sends a tier id (its "fast" config shows up as "priority"); Claude has a fastMode toggle.
const REPORTED_SERVICE_TIERS: ReadonlyMap<string, string> = new Map([
["default", "standard"],
["flex", "flex"],
["fast", "fast"],
["priority", "fast"],
["ultrafast", "ultrafast"],
]);

function selectedServiceTier(selection: ModelSelection): string | undefined {
const tier = getModelSelectionStringOptionValue(selection, "serviceTier");
if (tier !== undefined) return REPORTED_SERVICE_TIERS.get(tier);
const fastMode = getModelSelectionBooleanOptionValue(selection, "fastMode");
return fastMode === undefined ? undefined : fastMode ? "fast" : "standard";
}

/**
* Confirmed models across counted work. Unknown when any piece of work has no
* reported model, unless two distinct models were already seen.
Expand All @@ -130,6 +149,7 @@ function selectionProperties(
return defined({
model: selection === undefined ? undefined : label(selection.model),
effort: selection === undefined ? undefined : selectedEffort(selection),
serviceTier: selection === undefined ? undefined : selectedServiceTier(selection),
runtimeMode: context?.runtimeMode,
interactionMode: context?.interactionMode,
...context?.origin,
Expand Down
4 changes: 3 additions & 1 deletion docs/internals/product-analytics.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ is unknown or the ledger goes backwards, and background work that ends between
turns lands in the next one. Claude subagents have no turns of their own, so
this never overlaps child turns.

`model` and `effort` are what the run selected. `observedModel` and
`model`, `effort`, and `serviceTier` are what the run selected. `serviceTier`
is `standard`, `flex`, `fast`, or `ultrafast`, from Codex's tier option or
Claude's fast mode, and is absent when the selection does not set one. `observedModel` and
`mixedModels` appear only when the provider reported the models for the counted
work (Codex thread settings and reroutes, Claude root messages, OpenCode steps).
`mixedModels: false` without `reportVersion` was never checked. Model labels
Expand Down
Loading