From 7351558a76b44b9b4309f996ad301d9d1c208b19 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 12 Jul 2026 04:19:27 -0700 Subject: [PATCH 1/2] fix(ai-usage): record embeddings + self-host vision calls, fix advisory-routed model attribution Embeddings (src/review/rag.ts's inference calls) and self-host vision analysis (runVisualVisionForAdvisory/runScreenshotTableVisionForAdvisory's AI_VISION branches) were never recorded in ai_usage_events at all, leaving two whole AI features invisible to any usage dashboard. Also, advisory-routed calls (slop/e2e-test-gen/linked-issue- satisfaction/summaries routed through AI_ADVISORY) always recorded a stale hardcoded model label instead of the real model the provider reported, understating self-host Ollama usage and misattributing it to a legacy Workers-AI id. Fixes the root cause once in createOpenAiCompatibleAi (src/selfhost/ai.ts) so every caller gets real provider/model attribution for free, then updates each of the four affected features plus the new embeddings/vision recording call sites. --- src/queue/processors.ts | 71 +++++++++++++------ src/review/adapters.ts | 48 +++++++++++-- src/selfhost/ai.ts | 50 +++++++++++-- src/services/ai-e2e-test-gen.ts | 5 +- src/services/ai-slop.ts | 5 +- src/services/ai-summaries.ts | 24 ++++--- src/services/linked-issue-satisfaction-run.ts | 2 +- test/unit/ai-e2e-test-gen.test.ts | 27 +++++++ test/unit/ai-slop.test.ts | 25 +++++++ test/unit/ai-summaries.test.ts | 28 ++++++++ .../linked-issue-satisfaction-run.test.ts | 23 ++++++ test/unit/review-adapters.test.ts | 50 ++++++++++++- .../screenshot-table-vision-wiring.test.ts | 48 +++++++++++++ test/unit/selfhost-ai.test.ts | 39 ++++++++-- test/unit/visual-vision-wiring.test.ts | 46 ++++++++++++ 15 files changed, 441 insertions(+), 50 deletions(-) diff --git a/src/queue/processors.ts b/src/queue/processors.ts index 0311dac9c7..eca651be86 100644 --- a/src/queue/processors.ts +++ b/src/queue/processors.ts @@ -503,6 +503,7 @@ import type { LocalBranchAnalysisInput } from "../signals/local-branch"; import { callAiProvider, clampNumber, + coerceAiUsage, DEFAULT_BYOK_DAILY_REPO_LIMIT, hasPublicReviewAssessment, utcDayStartIso, @@ -6827,11 +6828,21 @@ type SelfHostVisionRunner = { run?: (model: string, options: Record { +/** Label recorded for a self-host vision call when the provider reports no usage/model at all (e.g. a + * malformed/non-JSON response) -- mirrors the same static-fallback convention as the other advisory AI + * features (WORKERS_SLOP_MODELS et al.), since there is no per-repo config field for this binding's model. */ +const SELF_HOST_VISION_MODEL_FALLBACK = "ollama:visual-vision"; + +async function runSelfHostVisualVision( + env: Env, + system: string, + user: string, + images: readonly AiContentBlock[], +): Promise<{ text: string | null; usage?: AiReviewActualUsage | undefined }> { const ai = env.AI_VISION as unknown as SelfHostVisionRunner | undefined; - if (!ai || typeof ai.run !== "function") return null; + if (!ai || typeof ai.run !== "function") return { text: null }; try { - const result = (await ai.run("visual-vision", { + const result = await ai.run("visual-vision", { messages: [ { role: "system", content: system }, { role: "user", content: [{ type: "text", text: user }, ...images] }, @@ -6842,10 +6853,11 @@ async function runSelfHostVisualVision(env: Env, system: string, user: string, i // concurrent load faster than the embed model does, degrading to latency collapse rather than a clean // OOM. Ignored by every non-Ollama provider (embeddings, subscription CLIs, Anthropic). providerOptions: { num_ctx: 4096 }, - })) as { response?: string } | null; - return result?.response?.trim() || null; + }); + const text = (result as { response?: string } | null)?.response?.trim() || null; + return { text, usage: coerceAiUsage(result) }; } catch { - return null; + return { text: null }; } } @@ -6976,22 +6988,25 @@ export async function runVisualVisionForAdvisory( return; } } else { - visionText = await runSelfHostVisualVision(env, VISUAL_VISION_SYSTEM_PROMPT, buildVisualVisionUserPrompt(visionGate.routes), images); + const selfHostResult = await runSelfHostVisualVision(env, VISUAL_VISION_SYSTEM_PROMPT, buildVisualVisionUserPrompt(visionGate.routes), images); + visionText = selfHostResult.text; + visionUsage = selfHostResult.usage; + if (!visionText) { + await recordVisualVisionUsage(env, args, null, "ok", "no usable output", visionUsage); + return; + } } - if (!visionText) return; const visionFindings = parseVisualVisionResponse(visionText); const findings = buildVisualRegressionFindings(visionFindings); args.advisory.findings.push(...findings); - if (visionProviderKey) { - await recordVisualVisionUsage( - env, - args, - visionProviderKey, - "ok", - findings.length > 0 ? `advisory findings (${findings.length})` : "no usable output", - visionUsage, - ); - } + await recordVisualVisionUsage( + env, + args, + visionProviderKey, + "ok", + findings.length > 0 ? `advisory findings (${findings.length})` : "no usable output", + visionUsage, + ); } catch (error) { console.log( JSON.stringify({ @@ -7007,7 +7022,7 @@ export async function runVisualVisionForAdvisory( async function recordVisualVisionUsage( env: Env, args: { repoFullName: string; pr: { number: number }; author: string | null }, - providerKey: { provider: string }, + providerKey: { provider: string } | null, status: string, detail: string, usage?: AiReviewActualUsage | undefined, @@ -7016,7 +7031,7 @@ async function recordVisualVisionUsage( feature: "visual_vision", actor: args.author ?? null, route: "github_app.visual_vision", - model: `byok:${providerKey.provider}`, + model: providerKey ? `byok:${providerKey.provider}` : (usage?.model ?? SELF_HOST_VISION_MODEL_FALLBACK), status, estimatedNeurons: 0, provider: usage?.provider, @@ -7033,7 +7048,7 @@ async function recordVisualVisionUsage( async function recordScreenshotTableVisionUsage( env: Env, args: { repoFullName: string; pr: { number: number }; author: string | null }, - providerKey: { provider: string }, + providerKey: { provider: string } | null, status: string, detail: string, usage?: AiReviewActualUsage | undefined, @@ -7042,7 +7057,7 @@ async function recordScreenshotTableVisionUsage( feature: "screenshot_table_vision", actor: args.author ?? null, route: "github_app.screenshot_table_vision", - model: `byok:${providerKey.provider}`, + model: providerKey ? `byok:${providerKey.provider}` : (usage?.model ?? SELF_HOST_VISION_MODEL_FALLBACK), status, estimatedNeurons: 0, provider: usage?.provider, @@ -7149,7 +7164,17 @@ export async function runScreenshotTableVisionForAdvisory( visionUsage, ); } else { - visionText = await runSelfHostVisualVision(env, SCREENSHOT_TABLE_VISION_SYSTEM_PROMPT, userPrompt, images); + const selfHostResult = await runSelfHostVisualVision(env, SCREENSHOT_TABLE_VISION_SYSTEM_PROMPT, userPrompt, images); + visionText = selfHostResult.text; + visionUsage = selfHostResult.usage; + await recordScreenshotTableVisionUsage( + env, + args, + null, + "ok", + visionText ? `advisory findings check (${gate.pairCount} pairs)` : "no usable output", + visionUsage, + ); } if (visionText) { const parsed = parseScreenshotTableVisionResponse(visionText, gate.pairCount); diff --git a/src/review/adapters.ts b/src/review/adapters.ts index fad8dc5a43..7f1b41b23e 100644 --- a/src/review/adapters.ts +++ b/src/review/adapters.ts @@ -13,6 +13,8 @@ // fail-safe on a missing vector/inference adapter ("no vector index → no RAG", "no AI → no context"), so the // modules NEVER throw — they degrade to no-context. Storage (D1 `DB`) is always present (the Worker cannot run // without it); its wrapper is a thin pass-through with the prepare→bind→all/first/run + batch surface RAG uses. +import { recordAiUsageEvent } from "../db/repositories"; +import { coerceAiUsage } from "../services/ai-review"; import { ragDimensionsFromEnv, ragEmbedBatchFromEnv, type InferenceAdapter, type RagInfra, type StorageAdapter, type VectorAdapter } from "./rag"; // ── Storage (D1 → StorageAdapter). Always present. A thin pass-through over `env.DB` — structurally the @@ -54,9 +56,47 @@ export function reviewVectorAdapter(vectorize: Vectorize): VectorAdapter { // ── Inference (the Ai-shaped adapter → InferenceAdapter). Feature-gated. Mirrors `ai.run(model, options)`; // the cast bridges the overloaded `run` signature to the portable single-signature shape. `ai` is -// Workers AI historically, and on self-host is the generic provider router (src/selfhost/ai.ts). ── -export function reviewInferenceAdapter(ai: Ai): InferenceAdapter { - return { run: (model, options) => (ai as unknown as { run(m: string, o: Record): Promise }).run(model, options) }; +// Workers AI historically, and on self-host is the generic provider router (src/selfhost/ai.ts). +// +// Also records every embedding call under the `embeddings` feature (2026-07 fix — previously +// RAG/embedding calls were never recorded in `ai_usage_events` at all). Self-host's `createOpenAiCompatibleAi` +// returns real `usage` (provider/model/tokens) for embeddings, so this is recorded for free via +// `coerceAiUsage`; Workers AI's binding has no such `usage` field, so the call is still recorded (feature + +// the model actually requested), just without token/cost detail. ── +export function reviewInferenceAdapter(env: Env, ai: Ai): InferenceAdapter { + const runner = ai as unknown as { run(m: string, o: Record): Promise }; + return { + run: async (model, options) => { + try { + const result = await runner.run(model, options); + const usage = coerceAiUsage(result); + await recordAiUsageEvent(env, { + feature: "embeddings", + route: "review.embeddings", + model: usage?.model ?? model, + provider: usage?.provider, + effort: usage?.effort, + status: "ok", + estimatedNeurons: 0, + inputTokens: usage?.inputTokens, + outputTokens: usage?.outputTokens, + totalTokens: usage?.totalTokens, + costUsd: usage?.costUsd, + }); + return result; + } catch (error) { + await recordAiUsageEvent(env, { + feature: "embeddings", + route: "review.embeddings", + model, + status: "error", + estimatedNeurons: 0, + detail: error instanceof Error ? error.message : "embedding_failed", + }); + throw error; + } + }, + }; } /** The infra bundle the ported review modules accept (`RagInfra`). Built from `Env`: @@ -75,6 +115,6 @@ export function createReviewAdapters(env: Env): RagInfra { // Embeddings use the DEDICATED embed provider (env.AI_EMBED) when configured — keeping the review chat chain // frontier-only — and fall back to env.AI otherwise (byte-identical to before). const embedAi = env.AI_EMBED ?? env.AI; - if (embedAi) infra.inference = reviewInferenceAdapter(embedAi); + if (embedAi) infra.inference = reviewInferenceAdapter(env, embedAi); return infra; } diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index eda2fd7a35..6d620d0679 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -295,11 +295,12 @@ export function createOpenAiCompatibleAi(opts: { async run(model, options) { // Embedding request — the core's embedTexts passes { text: string[] }; route to /embeddings (for RAG). if (Array.isArray(options.text)) { - if (options.text.length === 0) return { data: [] }; + const embedModel = opts.embedModel ?? "bge-m3"; + if (options.text.length === 0) return { data: [], usage: buildAiUsage({ provider: opts.providerName, model: embedModel, inputTokens: 0, totalTokens: 0 }) }; const res = await fetch(`${base}/embeddings`, { method: "POST", headers: headers(), - body: JSON.stringify({ model: opts.embedModel ?? "bge-m3", input: options.text }), + body: JSON.stringify({ model: embedModel, input: options.text }), signal: AbortSignal.timeout(120_000), }); // #4996: the error previously carried only the status code, with no detail from the response body -- @@ -311,8 +312,20 @@ export function createOpenAiCompatibleAi(opts: { const detail = await res.text().then((t) => t.slice(0, 300)).catch(() => ""); throw new Error(detail ? `ai_embed_http_${res.status}: ${detail}` : `ai_embed_http_${res.status}`); } - const json = (await res.json()) as { data?: Array<{ embedding: number[] }> }; - return { data: (json.data ?? []).map((d) => d.embedding) }; + // #ai-usage-embeddings: an OpenAI-compatible /embeddings response can carry a `usage` object + // (Ollama does); surface it so the RAG inference adapter (src/review/adapters.ts) can record real + // embedding usage instead of never tracking embeddings at all. Absent on providers that don't + // report it — inputTokens/totalTokens simply stay undefined, same fail-open shape as the chat path. + const json = (await res.json()) as { data?: Array<{ embedding: number[] }>; usage?: { prompt_tokens?: number; total_tokens?: number } }; + return { + data: (json.data ?? []).map((d) => d.embedding), + usage: buildAiUsage({ + provider: opts.providerName, + model: embedModel, + inputTokens: json.usage?.prompt_tokens, + totalTokens: json.usage?.total_tokens ?? json.usage?.prompt_tokens, + }), + }; } const repoOverride = opts.providerName ? resolveOpenAiCompatibleRepoOverride(opts.providerName, options) : undefined; const resolvedModel = resolveModel(firstConfigured(repoOverride, opts.model), model, opts.defaultModel ?? DEFAULT_OPENAI_COMPATIBLE_CHAT_MODEL); @@ -331,7 +344,11 @@ export function createOpenAiCompatibleAi(opts: { if (!res.ok) throw new Error(`ai_http_${res.status}`); const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> }; const usage = extractCliUsage(JSON.stringify(data)); - return { response: data.choices?.[0]?.message?.content ?? "", usage: { ...usage, model: usage.model ?? resolvedModel } }; + // #ai-usage-provider-attribution: extractCliUsage parses CLI-subprocess-style stdout markers, which an + // HTTP JSON chat-completions response never contains, so this path never has a real provider to read -- + // populate it from this adapter's own configured providerName (e.g. "ollama"), which every caller routed + // through env.AI_ADVISORY/AI_EMBED/AI_VISION previously had no way to attribute at all. + return { response: data.choices?.[0]?.message?.content ?? "", usage: buildAiUsage({ ...usage, model: usage.model ?? resolvedModel, provider: opts.providerName }) }; }, }; } @@ -569,6 +586,29 @@ export type AiUsage = CliUsage & { effort?: string; }; +/** Build an `AiUsage` object, omitting any field whose value is `undefined` -- `exactOptionalPropertyTypes` + * forbids assigning `undefined` to an optional property explicitly, it must simply be absent (mirrors the + * same pattern in src/review/adapters.ts's own `createReviewAdapters`). */ +function buildAiUsage(fields: { + provider?: string | undefined; + model?: string | undefined; + inputTokens?: number | undefined; + outputTokens?: number | undefined; + totalTokens?: number | undefined; + costUsd?: number | undefined; + effort?: string | undefined; +}): AiUsage { + const usage: AiUsage = {}; + if (fields.provider !== undefined) usage.provider = fields.provider; + if (fields.model !== undefined) usage.model = fields.model; + if (fields.inputTokens !== undefined) usage.inputTokens = fields.inputTokens; + if (fields.outputTokens !== undefined) usage.outputTokens = fields.outputTokens; + if (fields.totalTokens !== undefined) usage.totalTokens = fields.totalTokens; + if (fields.costUsd !== undefined) usage.costUsd = fields.costUsd; + if (fields.effort !== undefined) usage.effort = fields.effort; + return usage; +} + const INPUT_TOKEN_KEYS = ["input_tokens", "inputTokens", "prompt_tokens", "promptTokens"] as const; const OUTPUT_TOKEN_KEYS = ["output_tokens", "outputTokens", "completion_tokens", "completionTokens"] as const; const TOTAL_TOKEN_KEYS = ["total_tokens", "totalTokens"] as const; diff --git a/src/services/ai-e2e-test-gen.ts b/src/services/ai-e2e-test-gen.ts index 8d8abe7b2f..e8f2daa7c7 100644 --- a/src/services/ai-e2e-test-gen.ts +++ b/src/services/ai-e2e-test-gen.ts @@ -229,7 +229,10 @@ async function record( actor: input.actor ?? null, route: "github_app.ai_e2e_test_gen", // `byok:` so countByokAiEventsForRepoSince (model LIKE 'byok:%') counts it toward the cap. - model: input.providerKey ? `byok:${input.providerKey.provider}` : E2E_TEST_GEN_MODELS.join("+"), + // Non-BYOK: prefer the REAL model the provider reported (usage.model, populated for self-host CLI/HTTP + // providers including advisory-routed Ollama calls) over the hardcoded fallback label, which otherwise + // misrepresented every advisory-routed call as a legacy Workers-AI model id it never actually ran on. + model: input.providerKey ? `byok:${input.providerKey.provider}` : (usage?.model ?? E2E_TEST_GEN_MODELS.join("+")), status, estimatedNeurons, provider: usage?.provider, diff --git a/src/services/ai-slop.ts b/src/services/ai-slop.ts index 7241eab007..b37081120f 100644 --- a/src/services/ai-slop.ts +++ b/src/services/ai-slop.ts @@ -257,7 +257,10 @@ async function record( actor: input.actor ?? null, route: "github_app.ai_slop", // `byok:` so countByokAiEventsForRepoSince (model LIKE 'byok:%') counts it toward the cap. - model: input.providerKey ? `byok:${input.providerKey.provider}` : WORKERS_SLOP_MODELS.join("+"), + // Non-BYOK: prefer the REAL model the provider reported (usage.model, populated for self-host CLI/HTTP + // providers including advisory-routed Ollama calls) over the hardcoded fallback label, which otherwise + // misrepresented every advisory-routed call as a legacy Workers-AI model id it never actually ran on. + model: input.providerKey ? `byok:${input.providerKey.provider}` : (usage?.model ?? WORKERS_SLOP_MODELS.join("+")), status, estimatedNeurons, provider: usage?.provider, diff --git a/src/services/ai-summaries.ts b/src/services/ai-summaries.ts index af31ed97e0..69d4800847 100644 --- a/src/services/ai-summaries.ts +++ b/src/services/ai-summaries.ts @@ -73,29 +73,33 @@ export async function summarizeAgentBundleWithAi(env: Env, bundle: AgentRunBundl }); const rawText = extractAiText(response); const usage = coerceAiUsage(response); + // Prefer the real model the (possibly advisory-routed) provider actually reported over the static + // configured label, so ai_usage_events attributes advisory-routed calls to the real serving model + // instead of always recording the empty-string/legacy fallback (2026-07 fix). + const resolvedModel = usage?.model ?? model; if (!rawText) throw new Error("empty_ai_summary"); if (visibility === "public" && containsPublicForbiddenText(rawText)) { await recordAi(env, bundle, { feature: `agent_${visibility}_summary`, - model, + model: resolvedModel, status: "unsafe", estimatedNeurons, detail: "public summary failed sanitizer", usage, }); - return { status: "unsafe", model, estimatedNeurons, reason: "public summary failed sanitizer" }; + return { status: "unsafe", model: resolvedModel, estimatedNeurons, reason: "public summary failed sanitizer" }; } const text = sanitizeAiText(rawText, visibility); await recordAi(env, bundle, { feature: `agent_${visibility}_summary`, - model, + model: resolvedModel, status: "ok", estimatedNeurons, detail: "summary generated", metadata: { visibility }, usage, }); - return { status: "ok", model, estimatedNeurons, text }; + return { status: "ok", model: resolvedModel, estimatedNeurons, text }; } catch (error) { const reason = error instanceof Error ? error.message : "ai_summary_failed"; await recordAi(env, bundle, { @@ -329,14 +333,18 @@ export async function rewriteSignalBundleWithAi(env: Env, req: AiRewriteRequest) }); const rawText = extractAiText(response); const usage = coerceAiUsage(response); + // Prefer the real model the (possibly advisory-routed) provider actually reported over the static + // configured label, so ai_usage_events attributes advisory-routed calls to the real serving model + // instead of always recording the empty-string/legacy fallback (2026-07 fix). + const resolvedModel = usage?.model ?? model; if (!rawText) throw new Error("empty_ai_summary"); if (req.visibility === "public" && containsPublicForbiddenText(rawText)) { - await recordGenericAi(env, req, { model, status: "unsafe", estimatedNeurons, detail: "public summary failed sanitizer", usage }); - return { status: "unsafe", text: req.fallbackText, model, estimatedNeurons, reason: "public summary failed sanitizer" }; + await recordGenericAi(env, req, { model: resolvedModel, status: "unsafe", estimatedNeurons, detail: "public summary failed sanitizer", usage }); + return { status: "unsafe", text: req.fallbackText, model: resolvedModel, estimatedNeurons, reason: "public summary failed sanitizer" }; } const text = sanitizeAiText(rawText, req.visibility); - await recordGenericAi(env, req, { model, status: "ok", estimatedNeurons, detail: "summary generated", metadata: { visibility: req.visibility }, usage }); - return { status: "ok", text, model, estimatedNeurons }; + await recordGenericAi(env, req, { model: resolvedModel, status: "ok", estimatedNeurons, detail: "summary generated", metadata: { visibility: req.visibility }, usage }); + return { status: "ok", text, model: resolvedModel, estimatedNeurons }; } catch (error) { const reason = error instanceof Error ? error.message : "ai_summary_failed"; await recordGenericAi(env, req, { model, status: "error", estimatedNeurons: 0, detail: reason }); diff --git a/src/services/linked-issue-satisfaction-run.ts b/src/services/linked-issue-satisfaction-run.ts index f73a267bdc..0b3d8dcaa6 100644 --- a/src/services/linked-issue-satisfaction-run.ts +++ b/src/services/linked-issue-satisfaction-run.ts @@ -164,7 +164,7 @@ async function record( feature: "linked_issue_satisfaction", actor: input.actor ?? null, route: "github_app.linked_issue_satisfaction", - model: input.providerKey ? `byok:${input.providerKey.provider}` : LINKED_ISSUE_SATISFACTION_MODELS.join("+"), + model: input.providerKey ? `byok:${input.providerKey.provider}` : (usage?.model ?? LINKED_ISSUE_SATISFACTION_MODELS.join("+")), status, estimatedNeurons, provider: usage?.provider, diff --git a/test/unit/ai-e2e-test-gen.test.ts b/test/unit/ai-e2e-test-gen.test.ts index d291b88799..c916d3f1d0 100644 --- a/test/unit/ai-e2e-test-gen.test.ts +++ b/test/unit/ai-e2e-test-gen.test.ts @@ -8,6 +8,7 @@ import { runGittensoryE2eTestGeneration, type E2eTestGenInput, } from "../../src/services/ai-e2e-test-gen"; +import { BEST_REVIEW_MODELS, RELIABLE_FALLBACK_MODELS } from "../../src/services/ai-review"; import { recordAiUsageEvent } from "../../src/db/repositories"; import { upsertRepoFocusManifest } from "../../src/signals/focus-manifest-loader"; import type { FocusManifestReviewConfig } from "../../src/signals/focus-manifest"; @@ -321,6 +322,32 @@ describe("runGittensoryE2eTestGeneration — gating + fail-safe", () => { expect(row?.estimated_neurons).toBe(result.estimatedNeurons); }); + it("records the REAL reported model, not the hardcoded fallback label, when the provider reports one (2026-07 fix)", async () => { + const run = vi.fn(async () => ({ response: fenced(VALID_TEST_SOURCE), usage: { provider: "ollama", model: "qwen3:8b" } })); + const env = enabledEnv(run); + await cacheEmptyManifest(env); + const result = await runGittensoryE2eTestGeneration(env, baseInput); + expect(result).toMatchObject({ status: "ok" }); + + const row = await env.DB.prepare("select model, provider from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("ai_e2e_test_gen") + .first<{ model: string; provider: string | null }>(); + expect(row).toMatchObject({ model: "qwen3:8b", provider: "ollama" }); + }); + + it("falls back to the hardcoded model label when the provider reports no usage/model at all", async () => { + const run = vi.fn(async () => ({ response: fenced(VALID_TEST_SOURCE) })); + const env = enabledEnv(run); + await cacheEmptyManifest(env); + const result = await runGittensoryE2eTestGeneration(env, baseInput); + expect(result).toMatchObject({ status: "ok" }); + + const row = await env.DB.prepare("select model from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("ai_e2e_test_gen") + .first<{ model: string }>(); + expect(row?.model).toBe([BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]].join("+")); + }); + it("passes the AI_GATEWAY_ID through to the default-reviewer call when configured", async () => { let capturedExtra: unknown; const run = vi.fn(async (_model: string, _options: unknown, extra: unknown) => { diff --git a/test/unit/ai-slop.test.ts b/test/unit/ai-slop.test.ts index 1ef4df9e7a..71005ebe4a 100644 --- a/test/unit/ai-slop.test.ts +++ b/test/unit/ai-slop.test.ts @@ -7,6 +7,7 @@ import { } from "../../src/services/ai-slop"; import { evaluateGateCheck } from "../../src/rules/advisory"; import { buildAiReviewDiff, runAiSlopForAdvisory } from "../../src/queue/processors"; +import { BEST_REVIEW_MODELS, RELIABLE_FALLBACK_MODELS } from "../../src/services/ai-review"; import { getCachedAiSlopAdvisory, putCachedAiSlopAdvisory, recordAiUsageEvent, upsertRepositoryAiKey } from "../../src/db/repositories"; import { aiSlopCacheInputFingerprint } from "../../src/review/ai-slop-cache-input"; import type { Advisory, PullRequestFileRecord, RepositorySettings } from "../../src/types"; @@ -226,6 +227,30 @@ describe("runGittensoryAiSlopAdvisory gating + fail-safe", () => { expect(result.finding).toMatchObject({ code: AI_SLOP_FINDING_CODE, severity: "warning" }); }); + it("records the REAL reported model, not the hardcoded fallback label, when the provider reports one (2026-07 fix)", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }), usage: { provider: "ollama", model: "qwen3:8b" } })); + const env = enabledEnv(run); + const result = await runGittensoryAiSlopAdvisory(env, baseInput); + expect(result.status).toBe("ok"); + + const row = await env.DB.prepare("select model, provider from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("ai_slop_pr") + .first<{ model: string; provider: string | null }>(); + expect(row).toMatchObject({ model: "qwen3:8b", provider: "ollama" }); + }); + + it("falls back to the hardcoded model label when the provider reports no usage/model at all", async () => { + const run = vi.fn(async () => ({ response: slopJson({ band: "elevated" }) })); + const env = enabledEnv(run); + const result = await runGittensoryAiSlopAdvisory(env, baseInput); + expect(result.status).toBe("ok"); + + const row = await env.DB.prepare("select model from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("ai_slop_pr") + .first<{ model: string }>(); + expect(row?.model).toBe([BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]].join("+")); + }); + it("returns no finding when the model judges the change clean", async () => { const run = vi.fn(async () => ({ response: slopJson({ band: "clean", rationale: "genuine effort", signals: [] }) })); const result = await runGittensoryAiSlopAdvisory(enabledEnv(run), baseInput); diff --git a/test/unit/ai-summaries.test.ts b/test/unit/ai-summaries.test.ts index 73577f4b8f..157369dd68 100644 --- a/test/unit/ai-summaries.test.ts +++ b/test/unit/ai-summaries.test.ts @@ -70,6 +70,23 @@ describe("Workers AI summaries", () => { ); }); + it("records the REAL reported model, not the empty-string configured fallback, when the provider reports one (2026-07 fix)", async () => { + const run = vi.fn(async () => ({ response: "Advisory-routed summary.", usage: { provider: "ollama", model: "qwen3:8b" } })); + const env = createTestEnv({ + AI: { run } as unknown as Ai, + AI_SUMMARIES_ENABLED: "true", + AI_DAILY_NEURON_BUDGET: "10000", + }); + + const result = await summarizeAgentBundleWithAi(env, bundleFixture(), "private"); + + expect(result).toMatchObject({ status: "ok", model: "qwen3:8b" }); + const row = await env.DB.prepare("select model, provider from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("agent_private_summary") + .first<{ model: string; provider: string | null }>(); + expect(row).toMatchObject({ model: "qwen3:8b", provider: "ollama" }); + }); + it("applies the default daily neuron budget when AI_DAILY_NEURON_BUDGET is unset", async () => { const run = vi.fn(async () => ({ response: "Summary within default budget." })); const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true" }); @@ -332,6 +349,17 @@ describe("optional deterministic-summary rewrite layer", () => { expect(result.text).not.toBe(DETERMINISTIC_BODY); }); + it("records the REAL reported model, not the empty-string configured fallback, when the provider reports one (2026-07 fix)", async () => { + const run = vi.fn(async () => ({ response: "Advisory-routed rewrite.", usage: { provider: "ollama", model: "qwen3:8b" } })); + const env = publicEnv({}, run); + const result = await rewriteSignalBundleWithAi(env, rewriteReq()); + expect(result).toMatchObject({ status: "ok", model: "qwen3:8b" }); + const row = await env.DB.prepare("select model, provider from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("pr_intelligence_comment") + .first<{ model: string; provider: string | null }>(); + expect(row).toMatchObject({ model: "qwen3:8b", provider: "ollama" }); + }); + it("applies default model, output-token, and daily-budget configuration when env vars are unset", async () => { const run = vi.fn(async () => ({ response: "Default-config summary." })); const env = createTestEnv({ AI: { run } as unknown as Ai, AI_SUMMARIES_ENABLED: "true", AI_PUBLIC_COMMENTS_ENABLED: "true" }); diff --git a/test/unit/linked-issue-satisfaction-run.test.ts b/test/unit/linked-issue-satisfaction-run.test.ts index b0d51847a9..beb40883b1 100644 --- a/test/unit/linked-issue-satisfaction-run.test.ts +++ b/test/unit/linked-issue-satisfaction-run.test.ts @@ -1,5 +1,6 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runGittensoryLinkedIssueSatisfaction, type LinkedIssueSatisfactionRunInput } from "../../src/services/linked-issue-satisfaction-run"; +import { BEST_REVIEW_MODELS, RELIABLE_FALLBACK_MODELS } from "../../src/services/ai-review"; import { buildAiReviewDiff, processJob, runLinkedIssueSatisfactionForAdvisory } from "../../src/queue/processors"; import { evaluateGateCheck } from "../../src/rules/advisory"; import { @@ -254,6 +255,28 @@ describe("runGittensoryLinkedIssueSatisfaction gating + fail-safe", () => { expect(result.result).toBeNull(); }); + it("records the REAL reported model, not the hardcoded fallback label, when the provider reports one (2026-07 fix)", async () => { + const run = vi.fn(async () => ({ response: satisfactionJson({ status: "addressed" }), usage: { provider: "ollama", model: "qwen3:8b" } })); + const env = enabledEnv(run); + const result = await runGittensoryLinkedIssueSatisfaction(env, baseInput); + expect(result.status).toBe("ok"); + const row = await env.DB.prepare("select model, provider from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("linked_issue_satisfaction") + .first<{ model: string; provider: string | null }>(); + expect(row).toMatchObject({ model: "qwen3:8b", provider: "ollama" }); + }); + + it("falls back to the hardcoded model label when the provider reports no usage/model at all", async () => { + const run = vi.fn(async () => ({ response: satisfactionJson({ status: "addressed" }) })); + const env = enabledEnv(run); + const result = await runGittensoryLinkedIssueSatisfaction(env, baseInput); + expect(result.status).toBe("ok"); + const row = await env.DB.prepare("select model from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("linked_issue_satisfaction") + .first<{ model: string }>(); + expect(row?.model).toBe([BEST_REVIEW_MODELS[0], RELIABLE_FALLBACK_MODELS[0]].join("+")); + }); + it("records a null actor as null (not undefined) when the caller omits it", async () => { const run = vi.fn(async () => ({ response: satisfactionJson({ status: "addressed" }) })); const env = enabledEnv(run); diff --git a/test/unit/review-adapters.test.ts b/test/unit/review-adapters.test.ts index 112552562b..610a8094b8 100644 --- a/test/unit/review-adapters.test.ts +++ b/test/unit/review-adapters.test.ts @@ -5,6 +5,7 @@ import { reviewStorageAdapter, reviewVectorAdapter, } from "../../src/review/adapters"; +import { createTestEnv } from "../helpers/d1"; // ── Minimal Env stubs ───────────────────────────────────────────────────────────────────────────── // Only the bindings the factory touches (DB / VECTORIZE / AI) are stubbed; the factory is given an @@ -168,9 +169,56 @@ describe("reviewVectorAdapter: delegates to env.VECTORIZE", () => { describe("reviewInferenceAdapter: delegates to env.AI", () => { it("forwards run(model, options) to the AI binding and returns its result", async () => { const ai = aiStub(); - const adapter = reviewInferenceAdapter(ai as unknown as Ai); + const env = createTestEnv({ AI: ai as unknown as Ai }); + const adapter = reviewInferenceAdapter(env, ai as unknown as Ai); const out = await adapter.run("@cf/baai/bge-m3", { text: ["hello"] }); expect(ai.run).toHaveBeenCalledWith("@cf/baai/bge-m3", { text: ["hello"] }); expect(out).toEqual({ data: [[0.1, 0.2]] }); }); + + it("records the embedding call under the `embeddings` feature, using the real reported provider/model (2026-07 fix)", async () => { + const ai = { run: vi.fn(async () => ({ data: [[0.1, 0.2]], usage: { provider: "ollama", model: "bge-m3" } })) }; + const env = createTestEnv({ AI: ai as unknown as Ai }); + const adapter = reviewInferenceAdapter(env, ai as unknown as Ai); + await adapter.run("@cf/baai/bge-m3", { text: ["hello"] }); + const row = await env.DB.prepare("select feature, model, provider, status from ai_usage_events order by rowid desc limit 1").first<{ + feature: string; + model: string; + provider: string | null; + status: string; + }>(); + expect(row).toMatchObject({ feature: "embeddings", model: "bge-m3", provider: "ollama", status: "ok" }); + }); + + it("falls back to the requested model label when the provider reports no usage at all (e.g. the Workers AI binding)", async () => { + const ai = aiStub(); + const env = createTestEnv({ AI: ai as unknown as Ai }); + const adapter = reviewInferenceAdapter(env, ai as unknown as Ai); + await adapter.run("@cf/baai/bge-m3", { text: ["hello"] }); + const row = await env.DB.prepare("select feature, model, provider, status from ai_usage_events order by rowid desc limit 1").first<{ + feature: string; + model: string; + provider: string | null; + status: string; + }>(); + expect(row).toMatchObject({ feature: "embeddings", model: "@cf/baai/bge-m3", provider: null, status: "ok" }); + }); + + it("records a failed embedding call and RETHROWS (the caller's own fail-safe still sees the error)", async () => { + const ai = { + run: vi.fn(async () => { + throw new Error("embed_provider_unreachable"); + }), + }; + const env = createTestEnv({ AI: ai as unknown as Ai }); + const adapter = reviewInferenceAdapter(env, ai as unknown as Ai); + await expect(adapter.run("@cf/baai/bge-m3", { text: ["hello"] })).rejects.toThrow("embed_provider_unreachable"); + const row = await env.DB.prepare("select feature, model, status, detail from ai_usage_events order by rowid desc limit 1").first<{ + feature: string; + model: string; + status: string; + detail: string | null; + }>(); + expect(row).toMatchObject({ feature: "embeddings", model: "@cf/baai/bge-m3", status: "error", detail: "embed_provider_unreachable" }); + }); }); diff --git a/test/unit/screenshot-table-vision-wiring.test.ts b/test/unit/screenshot-table-vision-wiring.test.ts index 3a27ab86f5..ed1387e922 100644 --- a/test/unit/screenshot-table-vision-wiring.test.ts +++ b/test/unit/screenshot-table-vision-wiring.test.ts @@ -318,6 +318,54 @@ describe("runScreenshotTableVisionForAdvisory (#4366)", () => { ]); }); + it("records the self-host call under `screenshot_table_vision` with the REAL reported provider/model (2026-07 fix)", async () => { + const runMock = vi.fn(async () => ({ + response: findingsResponse([{ pairIndex: 1, body: "Looks like a different app entirely." }]), + usage: { provider: "ollama", model: "qwen3-vl:8b" }, + })); + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: runMock }; + stubShotsAndProvider(null); + const adv = findingsHolder(); + await runScreenshotTableVisionForAdvisory(env, { + mode: "live", + repoFullName, + pr, + prBody: tableBody(BEFORE_URL, AFTER_URL), + prTitle: "Redesign the nav bar", + author: "alice", + confirmedContributor: true, + settings: gateEnabledSettings({ aiReviewByok: false }), + advisory: adv, + }); + const row = await env.DB.prepare("select feature, model, provider, status from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("screenshot_table_vision") + .first<{ feature: string; model: string; provider: string | null; status: string }>(); + expect(row).toMatchObject({ feature: "screenshot_table_vision", model: "qwen3-vl:8b", provider: "ollama", status: "ok" }); + }); + + it("records a self-host call with no usable output under the fallback model label when the provider reports no usage", async () => { + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: vi.fn(async () => ({ response: " " })) }; + stubShotsAndProvider(null); + const adv = findingsHolder(); + await runScreenshotTableVisionForAdvisory(env, { + mode: "live", + repoFullName, + pr, + prBody: tableBody(BEFORE_URL, AFTER_URL), + prTitle: "Redesign the nav bar", + author: "alice", + confirmedContributor: true, + settings: gateEnabledSettings({ aiReviewByok: false }), + advisory: adv, + }); + const row = await env.DB.prepare("select feature, model, provider, status, detail from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("screenshot_table_vision") + .first<{ feature: string; model: string; provider: string | null; status: string; detail: string | null }>(); + expect(row).toMatchObject({ feature: "screenshot_table_vision", model: "ollama:visual-vision", provider: null, status: "ok", detail: "no usable output" }); + }); + it("does not let an unconfirmed contributor spend self-host vision resources unless all-authors is enabled", async () => { const runMock = vi.fn(async () => ({ response: findingsResponse([{ pairIndex: 1, body: "should not run" }]) })); const env = byokEnv(); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 617cbed8f8..293015f823 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -137,6 +137,17 @@ describe("createOpenAiCompatibleAi (#979)", () => { expect(first?.body.model).toBe("llama3.1"); }); + it("attributes usage.provider from its own configured providerName (#ai-usage-provider-attribution) since an HTTP chat-completions response never reports one itself", async () => { + vi.stubGlobal("fetch", vi.fn(async () => + new Response(JSON.stringify({ choices: [{ message: { content: "ok" } }] }), { status: 200 }), + )); + const withProvider = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", providerName: "ollama" }).run("m", { prompt: "x" }); + expect(withProvider.usage).toMatchObject({ provider: "ollama" }); + const withoutProvider = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { prompt: "x" }); + expect(withoutProvider.usage?.provider).toBeUndefined(); + expect("provider" in (withoutProvider.usage ?? {})).toBe(false); + }); + it("only sends an Authorization header when this OpenAI-compatible provider has its own apiKey", async () => { const authHeaders: Array = []; vi.stubGlobal("fetch", vi.fn(async (_u: string, init?: RequestInit) => { @@ -177,15 +188,31 @@ describe("createOpenAiCompatibleAi (#979)", () => { await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { prompt: "p" })).rejects.toThrow(/ai_http_500/); }); - it("routes an embedding request ({ text }) to /embeddings and returns { data }", async () => { + it("routes an embedding request ({ text }) to /embeddings and returns { data } plus a model/provider usage tag (no token counts, since this response has no usage object)", async () => { let url = ""; vi.stubGlobal("fetch", vi.fn(async (u: string) => { url = u; return new Response(JSON.stringify({ data: [{ embedding: [0.1, 0.2] }, { embedding: [0.3, 0.4] }] }), { status: 200 }); })); - const out = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3" }).run("@cf/baai/bge-m3", { text: ["a", "b"] }); + const out = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3", providerName: "ollama" }).run("@cf/baai/bge-m3", { text: ["a", "b"] }); expect(url).toBe("http://o/v1/embeddings"); - expect(out).toEqual({ data: [[0.1, 0.2], [0.3, 0.4]] }); + expect(out).toEqual({ data: [[0.1, 0.2], [0.3, 0.4]], usage: { provider: "ollama", model: "bge-m3" } }); + }); + + it("surfaces real token usage from an /embeddings response that reports one (#ai-usage-embeddings)", async () => { + vi.stubGlobal("fetch", vi.fn(async () => + new Response(JSON.stringify({ data: [{ embedding: [0.1] }], usage: { prompt_tokens: 42, total_tokens: 42 } }), { status: 200 }), + )); + const out = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3", providerName: "ollama" }).run("m", { text: ["hello"] }); + expect(out).toEqual({ data: [[0.1]], usage: { provider: "ollama", model: "bge-m3", inputTokens: 42, totalTokens: 42 } }); + }); + + it("falls back to prompt_tokens for totalTokens when an /embeddings response reports no total_tokens", async () => { + vi.stubGlobal("fetch", vi.fn(async () => + new Response(JSON.stringify({ data: [{ embedding: [0.1] }], usage: { prompt_tokens: 7 } }), { status: 200 }), + )); + const out = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3" }).run("m", { text: ["hello"] }); + expect(out).toEqual({ data: [[0.1]], usage: { model: "bge-m3", inputTokens: 7, totalTokens: 7 } }); }); it("throws on a non-OK embeddings response, including the response body detail (#4996: previously thrown away)", async () => { @@ -214,11 +241,11 @@ describe("createOpenAiCompatibleAi (#979)", () => { expect(error).toBe(`ai_embed_http_400: ${"x".repeat(300)}`); }); - it("empty text array returns { data: [] } without a fetch", async () => { + it("empty text array returns { data: [] } (plus a zero-token usage tag) without a fetch", async () => { const fetchMock = vi.fn(); vi.stubGlobal("fetch", fetchMock); - const result = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1" }).run("m", { text: [] }); - expect(result).toEqual({ data: [] }); + const result = await createOpenAiCompatibleAi({ baseUrl: "http://o/v1", embedModel: "bge-m3", providerName: "ollama" }).run("m", { text: [] }); + expect(result).toEqual({ data: [], usage: { provider: "ollama", model: "bge-m3", inputTokens: 0, totalTokens: 0 } }); expect(fetchMock).not.toHaveBeenCalled(); }); diff --git a/test/unit/visual-vision-wiring.test.ts b/test/unit/visual-vision-wiring.test.ts index dfbb9d3332..5e2e28cf88 100644 --- a/test/unit/visual-vision-wiring.test.ts +++ b/test/unit/visual-vision-wiring.test.ts @@ -697,6 +697,52 @@ describe("runVisualVisionForAdvisory: self-host local vision provider (#4335)", expect(adv.findings).toEqual([]); }); + it("records the self-host call under `visual_vision` with the REAL reported provider/model, not silently dropped (2026-07 fix)", async () => { + const env = byokEnv(); + const runMock = vi.fn(async () => ({ + response: findingsResponse([{ path: "/app", body: "Nav bar overlaps the logo." }]), + usage: { provider: "ollama", model: "qwen3-vl:8b" }, + })); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: runMock }; + stubShots(); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + mode: "live", + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + const row = await env.DB.prepare("select feature, model, provider, status from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("visual_vision") + .first<{ feature: string; model: string; provider: string | null; status: string }>(); + expect(row).toMatchObject({ feature: "visual_vision", model: "qwen3-vl:8b", provider: "ollama", status: "ok" }); + }); + + it("records a self-host call with no usable output under the fallback model label when the provider reports no usage", async () => { + const env = byokEnv(); + (env as unknown as { AI_VISION: unknown }).AI_VISION = { run: vi.fn(async () => ({ response: " " })) }; + stubShots(); + const adv = findingsHolder(); + await runVisualVisionForAdvisory(env, { + mode: "live", + repoFullName, + pr, + author: "alice", + confirmedContributor: true, + settings: byokSettings({ aiReviewByok: false }), + advisory: adv, + routes: selfHostVisionRoutes(), + }); + const row = await env.DB.prepare("select feature, model, provider, status, detail from ai_usage_events where feature = ? order by rowid desc limit 1") + .bind("visual_vision") + .first<{ feature: string; model: string; provider: string | null; status: string; detail: string | null }>(); + expect(row).toMatchObject({ feature: "visual_vision", model: "ollama:visual-vision", provider: null, status: "ok", detail: "no usable output" }); + }); + it("adds no finding when env.AI_VISION is present but has no callable .run (a malformed binding)", async () => { const env = byokEnv(); (env as unknown as { AI_VISION: unknown }).AI_VISION = {}; From 44358d30af670f3f38e800fe021e020ea9078dd9 Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Sun, 12 Jul 2026 04:40:44 -0700 Subject: [PATCH 2/2] test(ai-usage): close branch-coverage gaps in the embeddings/buildAiUsage patch adapters.ts's new embedding-error handler never exercised its non-Error-thrown branch, and buildAiUsage's model/costUsd/effort branches are structurally unreachable through its 3 real call sites (none ever pass costUsd/effort, and model is always defined) -- add a direct unit test via a new __selfHostAiInternals export, matching this repo's existing internals-export convention for testing private helpers. --- src/selfhost/ai.ts | 2 ++ test/unit/review-adapters.test.ts | 18 ++++++++++++++++++ test/unit/selfhost-ai.test.ts | 10 +++++++++- 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/src/selfhost/ai.ts b/src/selfhost/ai.ts index 6d620d0679..88254df71b 100644 --- a/src/selfhost/ai.ts +++ b/src/selfhost/ai.ts @@ -1446,3 +1446,5 @@ export function resolveAiReviewerPlan( export function withAdvisoryAiEnv(env: Env, useAdvisory: boolean): Env { return useAdvisory && env.AI_ADVISORY ? { ...env, AI: env.AI_ADVISORY } : env; } + +export const __selfHostAiInternals = { buildAiUsage }; diff --git a/test/unit/review-adapters.test.ts b/test/unit/review-adapters.test.ts index 610a8094b8..ca0897195f 100644 --- a/test/unit/review-adapters.test.ts +++ b/test/unit/review-adapters.test.ts @@ -221,4 +221,22 @@ describe("reviewInferenceAdapter: delegates to env.AI", () => { }>(); expect(row).toMatchObject({ feature: "embeddings", model: "@cf/baai/bge-m3", status: "error", detail: "embed_provider_unreachable" }); }); + + it("records a failed embedding call with the literal fallback detail when the provider throws a non-Error value", async () => { + const ai = { + run: vi.fn(async () => { + throw "not an Error instance"; + }), + }; + const env = createTestEnv({ AI: ai as unknown as Ai }); + const adapter = reviewInferenceAdapter(env, ai as unknown as Ai); + await expect(adapter.run("@cf/baai/bge-m3", { text: ["hello"] })).rejects.toBe("not an Error instance"); + const row = await env.DB.prepare("select feature, model, status, detail from ai_usage_events order by rowid desc limit 1").first<{ + feature: string; + model: string; + status: string; + detail: string | null; + }>(); + expect(row).toMatchObject({ feature: "embeddings", model: "@cf/baai/bge-m3", status: "error", detail: "embedding_failed" }); + }); }); diff --git a/test/unit/selfhost-ai.test.ts b/test/unit/selfhost-ai.test.ts index 293015f823..befaaed2a7 100644 --- a/test/unit/selfhost-ai.test.ts +++ b/test/unit/selfhost-ai.test.ts @@ -2,7 +2,7 @@ import { chmodSync, mkdirSync, mkdtempSync, readFileSync, writeFileSync } from " import { tmpdir } from "node:os"; import { delimiter, join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, codexErrorFromStdout, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, isAiProviderHealthy, markAiProviderUnhealthyAtBoot, resetAiProviderCircuitBreakerForTest, resetAiProviderHealthForTest, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveClaudeFirstOutputTimeoutMs, resolveCodexAuthPath, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveCodexFirstOutputTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, withAdvisoryAiEnv } from "../../src/selfhost/ai"; +import { assertNoLegacySharedAiEnv, buildProvider, claudeErrorStatus, codexErrorFromStdout, createAnthropicAi, createChainAi, createClaudeCodeAi, createCodexAi, createOpenAiCompatibleAi, createSelfHostAi, extractCliText, extractCliUsage, isAiProviderHealthy, markAiProviderUnhealthyAtBoot, resetAiProviderCircuitBreakerForTest, resetAiProviderHealthForTest, resolveAiReviewerPlan, resolveClaudeCliTimeoutMs, resolveClaudeFirstOutputTimeoutMs, resolveCodexAuthPath, resolveCodexCliTimeoutMs, resolveCodexEffort, resolveCodexFirstOutputTimeoutMs, resolveEffort, resolveModel, resolveProviderNames, resolveRequiredCliProviders, resolveSubscriptionCliPath, redactSecrets, routeProviders, shouldMarkAiProviderUnhealthyAtBoot, subscriptionCliEnv, withAdvisoryAiEnv, __selfHostAiInternals } from "../../src/selfhost/ai"; import { labelSelfHostReviewerModel, labelSelfHostReviewerModels } from "../../src/selfhost/ai-config"; import { renderMetrics, resetMetrics } from "../../src/selfhost/metrics"; @@ -215,6 +215,14 @@ describe("createOpenAiCompatibleAi (#979)", () => { expect(out).toEqual({ data: [[0.1]], usage: { model: "bge-m3", inputTokens: 7, totalTokens: 7 } }); }); + it("buildAiUsage omits every undefined field and includes every defined one (direct unit test — costUsd/effort/an-undefined-model are never both exercised through the 3 real call sites)", () => { + const { buildAiUsage } = __selfHostAiInternals; + expect(buildAiUsage({})).toEqual({}); + expect( + buildAiUsage({ provider: "ollama", model: "m", inputTokens: 1, outputTokens: 2, totalTokens: 3, costUsd: 0.5, effort: "medium" }), + ).toEqual({ provider: "ollama", model: "m", inputTokens: 1, outputTokens: 2, totalTokens: 3, costUsd: 0.5, effort: "medium" }); + }); + it("throws on a non-OK embeddings response, including the response body detail (#4996: previously thrown away)", async () => { vi.stubGlobal("fetch", vi.fn(async () => new Response("bad request: input exceeds max length", { status: 400 }))); await expect(createOpenAiCompatibleAi({ baseUrl: "http://x/v1" }).run("m", { text: ["a"] })).rejects.toThrow(