From 05a62f62b2a567d22e7321557f2ecf6d6fbb4d76 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Jul 2026 14:49:51 +0200 Subject: [PATCH 1/3] test(node): Fix flaky vercelai tool-error express test Co-Authored-By: Claude Opus 4.8 (1M context) --- .../suites/tracing/vercelai/test.ts | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index 0b91ce62a02c..bb567f74b394 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -349,9 +349,9 @@ describe('Vercel AI integration (v4)', () => { createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool in express server', async () => { let transactionEvent: Event | undefined; - let errorEvent: Event | undefined; + const errorEvents: Event[] = []; - const runner = createRunner() + const builder = createRunner() // In orchestrion mode the tool error is captured mid-request, so envelopes can arrive in any order. .unordered() .expect({ @@ -387,14 +387,30 @@ describe('Vercel AI integration (v4)', () => { }) .expect({ event: event => { - errorEvent = event; + errorEvents.push(event); }, - }) - .start(); + }); + + // In orchestrion mode the tool error surfaces twice: the channel subscriber captures the raw error + // (tagged with the tool identity), and the express error handler captures the wrapped + // `AI_ToolExecutionError`. Expect both so we can pick the tagged one regardless of arrival order. + if (orchestrion) { + builder.expect({ + event: event => { + errorEvents.push(event); + }, + }); + } + + const runner = builder.start(); await runner.makeRequest('get', '/test/error-in-tool', { expectError: true }); await runner.completed(); + const errorEvent = orchestrion + ? errorEvents.find(event => event.tags?.['vercel.ai.tool.name'] === 'getWeather') + : errorEvents[0]; + expect(transactionEvent).toBeDefined(); expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); From 5a82ecc75ef1dabf76dabc71aaed7988d3e0777c Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Jul 2026 15:18:55 +0200 Subject: [PATCH 2/3] fix(vercelai): Avoid double-capturing v4 tool errors in orchestrion mode Co-Authored-By: Claude Opus 4.8 (1M context) --- .../suites/tracing/vercelai/test.ts | 94 +++++++------------ .../vercel-ai-orchestrion-subscriber.ts | 54 +++++++++-- 2 files changed, 81 insertions(+), 67 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts index bb567f74b394..b759495545cc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/test.ts @@ -321,38 +321,32 @@ describe('Vercel AI integration (v4)', () => { expect(errorEvent).toBeDefined(); expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); - // Trace id is shared between the transaction and the tool error. - expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); - if (orchestrion) { - // The channel subscriber captures the raw tool error and tags it with the tool identity. - expect(errorEvent!.level).toBe('error'); - expect(errorEvent!.tags).toMatchObject({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }); - } else { - // The OTel processor surfaces the SDK's wrapped `AI_ToolExecutionError`. - expect(errorEvent!.exception?.values).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - type: 'AI_ToolExecutionError', - value: 'Error executing tool getWeather: Error in tool', - }), - ]), - ); - expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); - } + // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError`. The + // channel subscriber deliberately doesn't self-capture v4 tool errors (that would double-report + // alongside the bubbled error), so a single error event is produced in both modes. + expect(errorEvent!.exception?.values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool getWeather: Error in tool', + }), + ]), + ); + // Both paths stamp the operation's call-site span onto the bubbled error, so the global + // unhandled-rejection handler restores it and correlates the report to the transaction's root span. + expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); + expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); }); }); createEsmAndCjsTests(__dirname, 'scenario-error-in-tool-express.mjs', 'instrument.mjs', (createRunner, test) => { test('captures error in tool in express server', async () => { let transactionEvent: Event | undefined; - const errorEvents: Event[] = []; + let errorEvent: Event | undefined; - const builder = createRunner() - // In orchestrion mode the tool error is captured mid-request, so envelopes can arrive in any order. + const runner = createRunner() + // The error and transaction/span envelopes can arrive in either order, so assert content, not order. .unordered() .expect({ transaction: transaction => { @@ -387,30 +381,14 @@ describe('Vercel AI integration (v4)', () => { }) .expect({ event: event => { - errorEvents.push(event); - }, - }); - - // In orchestrion mode the tool error surfaces twice: the channel subscriber captures the raw error - // (tagged with the tool identity), and the express error handler captures the wrapped - // `AI_ToolExecutionError`. Expect both so we can pick the tagged one regardless of arrival order. - if (orchestrion) { - builder.expect({ - event: event => { - errorEvents.push(event); + errorEvent = event; }, - }); - } - - const runner = builder.start(); + }) + .start(); await runner.makeRequest('get', '/test/error-in-tool', { expectError: true }); await runner.completed(); - const errorEvent = orchestrion - ? errorEvents.find(event => event.tags?.['vercel.ai.tool.name'] === 'getWeather') - : errorEvents[0]; - expect(transactionEvent).toBeDefined(); expect(transactionEvent!.transaction).toBe('GET /test/error-in-tool'); expect(transactionEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); @@ -419,24 +397,18 @@ describe('Vercel AI integration (v4)', () => { expect(errorEvent!.tags).toMatchObject({ 'test-tag': 'test-value' }); expect(errorEvent!.contexts!.trace!.trace_id).toBe(transactionEvent!.contexts!.trace!.trace_id); - if (orchestrion) { - // The channel subscriber captures the raw tool error and tags it with the tool identity. - expect(errorEvent!.tags).toMatchObject({ - 'vercel.ai.tool.name': 'getWeather', - 'vercel.ai.tool.callId': 'call-1', - }); - } else { - // The OTel processor surfaces the SDK's wrapped `AI_ToolExecutionError`. - expect(errorEvent!.exception?.values).toEqual( - expect.arrayContaining([ - expect.objectContaining({ - type: 'AI_ToolExecutionError', - value: 'Error executing tool getWeather: Error in tool', - }), - ]), - ); - expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); - } + // The tool error bubbles out of the `ai` call as the SDK's wrapped `AI_ToolExecutionError` and is + // captured once by the express error handler — the channel subscriber deliberately doesn't + // self-capture v4 tool errors, so orchestrion and OTel produce the same single error event. + expect(errorEvent!.exception?.values).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + type: 'AI_ToolExecutionError', + value: 'Error executing tool getWeather: Error in tool', + }), + ]), + ); + expect(errorEvent!.contexts!.trace!.span_id).toBe(transactionEvent!.contexts!.trace!.span_id); }); }); diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts index 4548ecc73b35..9f9aa71edbaa 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -1,6 +1,13 @@ /* eslint-disable max-lines */ import type { Span } from '@sentry/core'; -import { debug, getActiveSpan, isObjectLike, SPAN_STATUS_ERROR, withActiveSpan } from '@sentry/core'; +import { + addNonEnumerableProperty, + debug, + getActiveSpan, + isObjectLike, + SPAN_STATUS_ERROR, + withActiveSpan, +} from '@sentry/core'; import { DEBUG_BUILD } from '../debug-build'; import { CHANNELS } from '../orchestrion/channels'; import { bindTracingChannelToSpan, type TracingChannelPayloadWithSpan } from '../tracing-channel'; @@ -100,6 +107,17 @@ const callIdBySpan = new WeakMap(); // child `generate_content` span (whose event would fall back to the global default). v7's channel forwards // these flags on every event, so this keeps v6 identical. const recordingBySpan = new WeakMap>(); +// Operation spans (v4) whose thrown tool errors propagate out of the `ai` call. On v4 a tool-`execute` +// rejection is wrapped in `AI_ToolExecutionError` and re-thrown, so it reaches the user's own error +// handling (or our global handlers) — capturing it at the tool span too would double-report it, matching +// how the OTel path leaves v4 tool errors to bubble. On v5 `executeTools` swallows the rejection into +// `tool-error` content, so it never surfaces and we must capture it ourselves (see `failSpan`). +const toolErrorsBubbleToCaller = new WeakSet(); +// The span active when each operation was invoked (its call site, e.g. the enclosing request/`main` span). +// When an operation's error bubbles out unhandled it reaches the global handler outside any span, so — as +// the OTel path does — we stamp this span onto the error (in `beforeSpanEnd`) so the unhandled-rejection +// handler can restore it and correlate the captured error to the operation's trace. +const callSiteSpanByOperationSpan = new WeakMap(); // The `experimental_telemetry` objects we swapped in to suppress `ai`'s native OTel spans (see // `suppressNativeTelemetry`). Our skip logic treats `isEnabled === false` as "user disabled telemetry, @@ -220,10 +238,16 @@ function bindOperation( // from the channels, so the SDK's would be duplicates. Reads above have already captured everything // we need off `telemetry`. suppressNativeTelemetry(callOptions, telemetry); + // The span active here is the operation's call site — `bindTracingChannelToSpan` only makes the + // operation span active *after* this returns — so this is the span we later restore for a bubbled error. + const callSiteSpan = getActiveSpan(); const span = createSpanFromMessage(message, options); if (span) { messages.set(data, message); operationSpans.add(span); + if (callSiteSpan) { + callSiteSpanByOperationSpan.set(span, callSiteSpan); + } // v6's `executeToolCall` span: tracked so the v5 tool-`execute` patch can recognize (and skip) // tool calls it runs, since that patch sees this span as its active parent (see `patchToolExecute`). if (message.type === 'executeTool') { @@ -247,6 +271,9 @@ function bindOperation( // the patch is a no-op for `embed`/`embedMany`. if (isObjectLike(callOptions.model) && callOptions.model.specificationVersion === 'v1') { patchModelMethods(callOptions.model as PatchableModel, options); + // v1 models are v4, where a thrown tool error bubbles out of the call rather than being swallowed + // into `tool-error` content — so its tool spans must not self-capture the error (see `failSpan`). + toolErrorsBubbleToCaller.add(span); } } return span; @@ -262,7 +289,16 @@ function bindOperation( return; } // The helper's `error` handler already set the span status; only enrich from a successful result. - if (!('error' in data)) { + if ('error' in data) { + // The error bubbles out of the `ai` call. If nothing handles it before our global handler, that + // handler runs outside any span — so stamp the operation's call-site span onto the error, letting + // the unhandled-rejection handler restore it and correlate the report to this trace. This mirrors + // the OTel path; it's a no-op when the error is handled earlier (e.g. a framework error handler). + const callSiteSpan = callSiteSpanByOperationSpan.get(span); + if (callSiteSpan && isObjectLike(data.error)) { + addNonEnumerableProperty(data.error, '_sentry_active_span', callSiteSpan); + } + } else { // v6's `executeToolCall` returns the tool result/error object directly, whereas the shared core // (matching v7) expects it nested under `output`; wrap it so tool-error detection works. message.result = message.type === 'executeTool' ? { output: data.result } : data.result; @@ -586,11 +622,17 @@ function patchToolExecute( return original.apply(this, [input, ...rest]); } - // v5's `executeTools` catches a thrown tool error and turns it into `tool-error` content rather - // than rejecting, so the user never sees a rejection — we must capture it here. Rethrow so - // `executeTools` still produces its `tool-error` result and the operation continues normally. + // v5's `executeTools` catches a thrown tool error and turns it into `tool-error` content rather than + // rejecting, so the user never sees a rejection — we must capture it here. On v4 the rejection bubbles + // out of the call and reaches the user's error handling (or our global handlers), so we only mark the + // span and leave the capture to them, avoiding a duplicate report. Rethrow either way so `executeTools` + // still produces its `tool-error` result (v5) and the rejection propagates (v4). const failSpan = (error: unknown): never => { - captureToolError(span, message, error); + if (toolErrorsBubbleToCaller.has(parent)) { + span.setStatus({ code: SPAN_STATUS_ERROR, message: error instanceof Error ? error.message : 'tool_error' }); + } else { + captureToolError(span, message, error); + } span.end(); throw error; }; From 650005a151c07c99b492c7f2c7c99efb45aec652 Mon Sep 17 00:00:00 2001 From: Francesco Novy Date: Wed, 15 Jul 2026 15:33:17 +0200 Subject: [PATCH 3/3] ref(vercelai): Consolidate per-operation error info into one map Co-Authored-By: Claude Opus 4.8 (1M context) --- .../vercel-ai-orchestrion-subscriber.ts | 53 ++++++++++--------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts index 9f9aa71edbaa..b41290b281e9 100644 --- a/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts +++ b/packages/server-utils/src/vercel-ai/vercel-ai-orchestrion-subscriber.ts @@ -107,17 +107,21 @@ const callIdBySpan = new WeakMap(); // child `generate_content` span (whose event would fall back to the global default). v7's channel forwards // these flags on every event, so this keeps v6 identical. const recordingBySpan = new WeakMap>(); -// Operation spans (v4) whose thrown tool errors propagate out of the `ai` call. On v4 a tool-`execute` -// rejection is wrapped in `AI_ToolExecutionError` and re-thrown, so it reaches the user's own error -// handling (or our global handlers) — capturing it at the tool span too would double-report it, matching -// how the OTel path leaves v4 tool errors to bubble. On v5 `executeTools` swallows the rejection into -// `tool-error` content, so it never surfaces and we must capture it ourselves (see `failSpan`). -const toolErrorsBubbleToCaller = new WeakSet(); -// The span active when each operation was invoked (its call site, e.g. the enclosing request/`main` span). -// When an operation's error bubbles out unhandled it reaches the global handler outside any span, so — as -// the OTel path does — we stamp this span onto the error (in `beforeSpanEnd`) so the unhandled-rejection -// handler can restore it and correlate the captured error to the operation's trace. -const callSiteSpanByOperationSpan = new WeakMap(); +interface OperationErrorInfo { + // The span active when the operation was invoked (its call site, e.g. the enclosing request/`main` span). + // When an operation's error bubbles out unhandled it reaches the global handler outside any span, so — as + // the OTel path does — we stamp this span onto the error (in `beforeSpanEnd`) so the unhandled-rejection + // handler can restore it and correlate the captured error to the operation's trace. + callSiteSpan: Span | undefined; + // Whether a thrown tool error bubbles out of the `ai` call (v4). On v4 a tool-`execute` rejection is + // wrapped in `AI_ToolExecutionError` and re-thrown, so it reaches the user's own error handling (or our + // global handlers) — capturing it at the tool span too would double-report it, matching how the OTel path + // leaves v4 tool errors to bubble. On v5 `executeTools` swallows the rejection into `tool-error` content, + // so it never surfaces and we must capture it ourselves (see `failSpan`). + toolErrorsBubbleToCaller: boolean; +} +// Per-operation error-handling info, keyed by the operation span (see `OperationErrorInfo`). +const operationErrorInfoBySpan = new WeakMap(); // The `experimental_telemetry` objects we swapped in to suppress `ai`'s native OTel spans (see // `suppressNativeTelemetry`). Our skip logic treats `isEnabled === false` as "user disabled telemetry, @@ -245,9 +249,10 @@ function bindOperation( if (span) { messages.set(data, message); operationSpans.add(span); - if (callSiteSpan) { - callSiteSpanByOperationSpan.set(span, callSiteSpan); - } + // `'v1'` models are v4 (v5/v6 are `'v2'`), where a thrown tool error bubbles out of the call rather + // than being swallowed into `tool-error` content — so its tool spans must not self-capture the error. + const isV4 = isObjectLike(callOptions.model) && callOptions.model.specificationVersion === 'v1'; + operationErrorInfoBySpan.set(span, { callSiteSpan, toolErrorsBubbleToCaller: isV4 }); // v6's `executeToolCall` span: tracked so the v5 tool-`execute` patch can recognize (and skip) // tool calls it runs, since that patch sees this span as its active parent (see `patchToolExecute`). if (message.type === 'executeTool') { @@ -263,17 +268,13 @@ function bindOperation( if (isObjectLike(callOptions.tools)) { patchOperationTools(callOptions.tools, options); } - // v4 has no `resolveLanguageModel` chokepoint — the passed `LanguageModelV1` - // (`specificationVersion: 'v1'`) is called directly — so patch its `doGenerate`/`doStream` - // here, at the operation start, instead. v5/v6 models are `'v2'` and are patched via - // `resolveLanguageModel`, so restricting to `'v1'` keeps this strictly additive and avoids - // double-patching. Embedding models are also `'v1'` but expose no `doGenerate`/`doStream`, so - // the patch is a no-op for `embed`/`embedMany`. - if (isObjectLike(callOptions.model) && callOptions.model.specificationVersion === 'v1') { + // v4 has no `resolveLanguageModel` chokepoint — the passed `LanguageModelV1` is called directly — so + // patch its `doGenerate`/`doStream` here, at the operation start, instead. v5/v6 models are patched via + // `resolveLanguageModel`, so restricting to v4 keeps this strictly additive and avoids double-patching. + // Embedding models are also `'v1'` but expose no `doGenerate`/`doStream`, so the patch is a no-op for + // `embed`/`embedMany`. + if (isV4) { patchModelMethods(callOptions.model as PatchableModel, options); - // v1 models are v4, where a thrown tool error bubbles out of the call rather than being swallowed - // into `tool-error` content — so its tool spans must not self-capture the error (see `failSpan`). - toolErrorsBubbleToCaller.add(span); } } return span; @@ -294,7 +295,7 @@ function bindOperation( // handler runs outside any span — so stamp the operation's call-site span onto the error, letting // the unhandled-rejection handler restore it and correlate the report to this trace. This mirrors // the OTel path; it's a no-op when the error is handled earlier (e.g. a framework error handler). - const callSiteSpan = callSiteSpanByOperationSpan.get(span); + const callSiteSpan = operationErrorInfoBySpan.get(span)?.callSiteSpan; if (callSiteSpan && isObjectLike(data.error)) { addNonEnumerableProperty(data.error, '_sentry_active_span', callSiteSpan); } @@ -628,7 +629,7 @@ function patchToolExecute( // span and leave the capture to them, avoiding a duplicate report. Rethrow either way so `executeTools` // still produces its `tool-error` result (v5) and the rejection propagates (v4). const failSpan = (error: unknown): never => { - if (toolErrorsBubbleToCaller.has(parent)) { + if (operationErrorInfoBySpan.get(parent)?.toolErrorsBubbleToCaller) { span.setStatus({ code: SPAN_STATUS_ERROR, message: error instanceof Error ? error.message : 'tool_error' }); } else { captureToolError(span, message, error);