diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cd9d7619..e105f22da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Deprecated `GOOGLE_VERTEX_THINKING_BUDGET_TOKENS` environment variable in favor of per-model `thinkingBudget` config. [#1110](https://github.com/sourcebot-dev/sourcebot/pull/1110) - Removed `GOOGLE_VERTEX_INCLUDE_THOUGHTS` environment variable. Thoughts are now always included. [#1110](https://github.com/sourcebot-dev/sourcebot/pull/1110) +- Renamed and consolidated PostHog chat events (`wa_chat_thread_created` -> `ask_thread_created`, `wa_chat_message_sent` -> `ask_message_sent`, `wa_chat_tool_used` -> `tool_used`), added unified `tool_used` tracking across the ask agent and MCP server, and removed the redundant `api_code_search_request` event. [#1111](https://github.com/sourcebot-dev/sourcebot/pull/1111) ## [4.16.8] - 2026-04-09 diff --git a/CLAUDE.md b/CLAUDE.md index 943898eae..21f044578 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -57,6 +57,12 @@ if (!value) { return; } if (condition) doSomething(); ``` +## PostHog Event Naming + +- The `wa_` prefix is reserved for events that can ONLY be fired from the web app (e.g., `wa_login_with_github`, `wa_chat_feedback_submitted`). +- Events fired from multiple sources (web app, MCP server, API) must NOT use the `wa_` prefix (e.g., `ask_message_sent`, `tool_used`). +- Multi-source events should include a `source` property to identify the origin (e.g., `'sourcebot-web-client'`, `'sourcebot-mcp-server'`, `'sourcebot-ask-agent'`). + ## Tailwind CSS Use Tailwind color classes directly instead of CSS variable syntax: diff --git a/packages/web/src/app/api/(server)/chat/route.ts b/packages/web/src/app/api/(server)/chat/route.ts index 2ae2f9700..4c0b12819 100644 --- a/packages/web/src/app/api/(server)/chat/route.ts +++ b/packages/web/src/app/api/(server)/chat/route.ts @@ -91,12 +91,15 @@ export const POST = apiHandler(async (req: NextRequest) => { return []; }))).flat(); - await captureEvent('wa_chat_message_sent', { + const source = req.headers.get('X-Sourcebot-Client-Source') ?? undefined; + + await captureEvent('ask_message_sent', { chatId: id, messageCount: messages.length, selectedReposCount: expandedRepos.length, + source, ...(env.EXPERIMENT_ASK_GH_ENABLED === 'true' ? { selectedRepos: expandedRepos } : {}), - } ); + }); const stream = await createMessageStream({ chatId: id, diff --git a/packages/web/src/app/api/(server)/search/route.ts b/packages/web/src/app/api/(server)/search/route.ts index d13aae3ba..e15184297 100644 --- a/packages/web/src/app/api/(server)/search/route.ts +++ b/packages/web/src/app/api/(server)/search/route.ts @@ -2,7 +2,6 @@ import { search, searchRequestSchema } from "@/features/search"; import { apiHandler } from "@/lib/apiHandler"; -import { captureEvent } from "@/lib/posthog"; import { requestBodySchemaValidationError, serviceErrorResponse } from "@/lib/serviceError"; import { isServiceError } from "@/lib/utils"; import { NextRequest } from "next/server"; @@ -21,12 +20,6 @@ export const POST = apiHandler(async (request: NextRequest) => { ...options } = parsed.data; - const source = request.headers.get('X-Sourcebot-Client-Source') ?? 'unknown'; - await captureEvent('api_code_search_request', { - source, - type: 'blocking', - }); - const response = await search({ queryType: 'string', query, diff --git a/packages/web/src/app/api/(server)/stream_search/route.ts b/packages/web/src/app/api/(server)/stream_search/route.ts index 4caeba5b4..5c7ab92cb 100644 --- a/packages/web/src/app/api/(server)/stream_search/route.ts +++ b/packages/web/src/app/api/(server)/stream_search/route.ts @@ -2,7 +2,6 @@ import { streamSearch, searchRequestSchema } from '@/features/search'; import { apiHandler } from '@/lib/apiHandler'; -import { captureEvent } from '@/lib/posthog'; import { requestBodySchemaValidationError, serviceErrorResponse } from '@/lib/serviceError'; import { isServiceError } from '@/lib/utils'; import { NextRequest } from 'next/server'; @@ -20,12 +19,6 @@ export const POST = apiHandler(async (request: NextRequest) => { ...options } = parsed.data; - const source = request.headers.get('X-Sourcebot-Client-Source') ?? 'unknown'; - await captureEvent('api_code_search_request', { - source, - type: 'streamed', - }); - const stream = await streamSearch({ queryType: 'string', query, diff --git a/packages/web/src/features/chat/actions.ts b/packages/web/src/features/chat/actions.ts index 3762e88c3..dac24d155 100644 --- a/packages/web/src/features/chat/actions.ts +++ b/packages/web/src/features/chat/actions.ts @@ -49,9 +49,10 @@ export const createChat = async ({ source }: { source?: string } = {}) => sew(() }); } - await captureEvent('wa_chat_thread_created', { + await captureEvent('ask_thread_created', { chatId: chat.id, isAnonymous: isGuestUser, + source, }); return { diff --git a/packages/web/src/features/chat/agent.ts b/packages/web/src/features/chat/agent.ts index 2e2676299..784ca5344 100644 --- a/packages/web/src/features/chat/agent.ts +++ b/packages/web/src/features/chat/agent.ts @@ -1,7 +1,6 @@ import { SBChatMessage, SBChatMessageMetadata } from "@/features/chat/types"; import { getAnswerPartFromAssistantMessage } from "@/features/chat/utils"; import { getFileSource } from '@/features/git'; -import { captureEvent } from "@/lib/posthog"; import { isServiceError } from "@/lib/utils"; import { LanguageModelV3 as AISDKLanguageModelV3 } from "@ai-sdk/provider"; import { ProviderOptions } from "@ai-sdk/provider-utils"; @@ -210,13 +209,7 @@ const createAgentStream = async ({ ], toolChoice: "auto", onStepFinish: ({ toolResults }) => { - toolResults.forEach(({ toolName, output, dynamic }) => { - captureEvent('wa_chat_tool_used', { - chatId, - toolName, - success: !isServiceError(output), - }); - + toolResults.forEach(({ output, dynamic }) => { if (dynamic || isServiceError(output)) { return; } diff --git a/packages/web/src/features/chat/components/chatThread/chatThread.tsx b/packages/web/src/features/chat/components/chatThread/chatThread.tsx index 1cdf8ef0b..f60d281b7 100644 --- a/packages/web/src/features/chat/components/chatThread/chatThread.tsx +++ b/packages/web/src/features/chat/components/chatThread/chatThread.tsx @@ -102,6 +102,9 @@ export const ChatThread = ({ messages: initialMessages, transport: new DefaultChatTransport({ api: '/api/chat', + headers: { + 'X-Sourcebot-Client-Source': 'sourcebot-web-client', + }, }), onData: (dataPart) => { // Keeps sources added by the assistant in sync. diff --git a/packages/web/src/features/mcp/askCodebase.ts b/packages/web/src/features/mcp/askCodebase.ts index 6ca9c26a2..6451e680f 100644 --- a/packages/web/src/features/mcp/askCodebase.ts +++ b/packages/web/src/features/mcp/askCodebase.ts @@ -86,9 +86,10 @@ export const askCodebase = (params: AskCodebaseParams): Promise r.value) } : {}), diff --git a/packages/web/src/features/mcp/server.ts b/packages/web/src/features/mcp/server.ts index 1de4efee4..ec4103f36 100644 --- a/packages/web/src/features/mcp/server.ts +++ b/packages/web/src/features/mcp/server.ts @@ -2,6 +2,7 @@ import { languageModelInfoSchema, } from '@/features/chat/types'; import { askCodebase } from '@/features/mcp/askCodebase'; +import { captureEvent } from '@/lib/posthog'; import { isServiceError } from '@/lib/utils'; import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { ChatVisibility } from '@sourcebot/db'; @@ -57,6 +58,11 @@ export async function createMcpServer(): Promise { }, async () => { const models = await getConfiguredLanguageModelsInfo(); + captureEvent('tool_used', { + toolName: 'list_language_models', + source: 'sourcebot-mcp-server', + success: true, + }); return { content: [{ type: "text", text: JSON.stringify(models) }] }; } ); @@ -101,11 +107,22 @@ export async function createMcpServer(): Promise { }); if (isServiceError(result)) { + captureEvent('tool_used', { + toolName: 'ask_codebase', + source: 'sourcebot-mcp-server', + success: false, + }); return { content: [{ type: "text", text: `Failed to ask codebase: ${result.message}` }], }; } + captureEvent('tool_used', { + toolName: 'ask_codebase', + source: 'sourcebot-mcp-server', + success: true, + }); + const formattedResponse = dedent` ${result.answer} diff --git a/packages/web/src/features/tools/adapters.ts b/packages/web/src/features/tools/adapters.ts index 5b2ac3809..2d3a4142e 100644 --- a/packages/web/src/features/tools/adapters.ts +++ b/packages/web/src/features/tools/adapters.ts @@ -1,6 +1,7 @@ import { tool } from "ai"; import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; import { z } from "zod"; +import { captureEvent } from "@/lib/posthog"; import { ToolContext, ToolDefinition } from "./types"; export function toVercelAITool( @@ -11,7 +12,21 @@ export function toVercelAITool def.execute(input, context), + execute: async (input) => { + let success = true; + try { + return await def.execute(input, context); + } catch (error) { + success = false; + throw error; + } finally { + captureEvent('tool_used', { + toolName: def.name, + source: context.source ?? 'unknown', + success, + }); + } + }, toModelOutput: ({ output }) => ({ type: "content", value: [{ type: "text", text: output.output }], @@ -38,13 +53,21 @@ export function registerMcpTool { + let success = true; try { const parsed = def.inputSchema.parse(input); const result = await def.execute(parsed, context); return { content: [{ type: "text" as const, text: result.output }] }; } catch (error) { + success = false; const message = error instanceof Error ? error.message : String(error); return { content: [{ type: "text" as const, text: `Tool "${def.name}" failed: ${message}` }], isError: true }; + } finally { + captureEvent('tool_used', { + toolName: def.name, + source: context.source ?? 'unknown', + success, + }); } }, ); diff --git a/packages/web/src/lib/apiHandler.ts b/packages/web/src/lib/apiHandler.ts index 65c76f116..0015a805a 100644 --- a/packages/web/src/lib/apiHandler.ts +++ b/packages/web/src/lib/apiHandler.ts @@ -47,9 +47,7 @@ export function apiHandler( const source = request.headers.get('X-Sourcebot-Client-Source') ?? 'unknown'; // Fire and forget - don't await to avoid blocking the request - captureEvent('api_request', { path, method, source }).catch(() => { - // Silently ignore tracking errors - }); + captureEvent('api_request', { path, method, source }); } // Call the original handler with all arguments diff --git a/packages/web/src/lib/posthog.ts b/packages/web/src/lib/posthog.ts index 33d335fb8..4c4a56c0d 100644 --- a/packages/web/src/lib/posthog.ts +++ b/packages/web/src/lib/posthog.ts @@ -1,11 +1,12 @@ import { PostHog } from 'posthog-node' -import { env, SOURCEBOT_VERSION } from '@sourcebot/shared' +import { createLogger, env, SOURCEBOT_VERSION } from '@sourcebot/shared' import { RequestCookies } from 'next/dist/compiled/@edge-runtime/cookies'; import * as Sentry from "@sentry/nextjs"; import { PosthogEvent, PosthogEventMap } from './posthogEvents'; import { cookies, headers } from 'next/headers'; -import { auth } from '@/auth'; -import { getVerifiedApiObject } from '@/middleware/withAuth'; +import { getAuthenticatedUser } from '@/middleware/withAuth'; + +const logger = createLogger('posthog'); /** * @note: This is a subset of the properties stored in the @@ -53,28 +54,19 @@ const getPostHogCookie = (cookieStore: Pick): PostHogCook * Attempts to retrieve the distinct id of the current user. */ export const tryGetPostHogDistinctId = async () => { - // First, attempt to retrieve the distinct id from the cookie. + // First, attempt to retrieve the distinct id from the PostHog cookie + // (set by the client-side PostHog SDK). This preserves identity + // continuity between client-side and server-side events. const cookieStore = await cookies(); const cookie = getPostHogCookie(cookieStore); if (cookie) { return cookie.distinct_id; } - // Next, from the session. - const session = await auth(); - if (session) { - return session.user.id; - } - - // Finally, from the api key. - const headersList = await headers(); - const apiKeyString = headersList.get("X-Sourcebot-Api-Key") ?? undefined; - if (!apiKeyString) { - return undefined; - } - - const apiKey = await getVerifiedApiObject(apiKeyString); - return apiKey?.createdById; + // Fall back to the authenticated user's ID. This covers all auth + // methods: session cookies, OAuth Bearer tokens, and API keys. + const authResult = await getAuthenticatedUser(); + return authResult?.user.id; } export const createPostHogClient = async () => { @@ -88,24 +80,28 @@ export const createPostHogClient = async () => { } export async function captureEvent(event: E, properties: PosthogEventMap[E]) { - if (env.SOURCEBOT_TELEMETRY_DISABLED === 'true') { - return; - } + try { + if (env.SOURCEBOT_TELEMETRY_DISABLED === 'true') { + return; + } - const distinctId = await tryGetPostHogDistinctId(); - const posthog = await createPostHogClient(); - - const headersList = await headers(); - const host = headersList.get("host") ?? undefined; - - posthog.capture({ - event, - properties: { - ...properties, - sourcebot_version: SOURCEBOT_VERSION, - install_id: env.SOURCEBOT_INSTALL_ID, - $host: host, - }, - distinctId, - }); + const distinctId = await tryGetPostHogDistinctId(); + const posthog = await createPostHogClient(); + + const headersList = await headers(); + const host = headersList.get("host") ?? undefined; + + posthog.capture({ + event, + properties: { + ...properties, + sourcebot_version: SOURCEBOT_VERSION, + install_id: env.SOURCEBOT_INSTALL_ID, + $host: host, + }, + distinctId, + }); + } catch (error) { + logger.error('Failed to capture PostHog event:', error); + } } \ No newline at end of file diff --git a/packages/web/src/lib/posthogEvents.ts b/packages/web/src/lib/posthogEvents.ts index 6eee712d6..94d3d9b88 100644 --- a/packages/web/src/lib/posthogEvents.ts +++ b/packages/web/src/lib/posthogEvents.ts @@ -159,14 +159,16 @@ export type PosthogEventMap = { chatId: string, messageId: string, }, - wa_chat_thread_created: { + ask_thread_created: { chatId: string, isAnonymous: boolean, + source?: string, }, - wa_chat_message_sent: { + ask_message_sent: { chatId: string, messageCount: number, selectedReposCount: number, + source?: string, /** * @note this field will only be populated when * the EXPERIMENT_ASK_GH_ENABLED environment variable @@ -174,9 +176,9 @@ export type PosthogEventMap = { */ selectedRepos?: string[], }, - wa_chat_tool_used: { - chatId: string, + tool_used: { toolName: string, + source: string, success: boolean, }, wa_chat_share_dialog_opened: { @@ -286,10 +288,6 @@ export type PosthogEventMap = { ////////////////////////////////////////////////////////////////// wa_repo_not_found_for_zoekt_file: {}, ////////////////////////////////////////////////////////////////// - api_code_search_request: { - source: string; - type: 'streamed' | 'blocking'; - }, api_request: { path: string; source: string;