From 5c51db64db22390d023f002d9b510b2a0222b811 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 12:52:10 -0700 Subject: [PATCH 1/2] perf(tests): cut per-file import graphs in apps/sim and shard the suite across CI runners --- .github/workflows/test-build.yml | 67 ++++++ .../app/api/copilot/tools/execute/route.ts | 2 +- .../mention/mention-icon.test.ts | 5 +- apps/sim/blocks/blocks/harmonic.test.ts | 13 +- apps/sim/blocks/blocks/outlook.test.ts | 11 +- apps/sim/blocks/brand-icon.test.tsx | 3 + .../lib/billing/core/limit-notifications.ts | 7 +- apps/sim/lib/billing/core/usage.ts | 31 ++- .../catalog/projection/catalog-sweep.test.ts | 7 + .../sim/lib/copilot/request/tools/executor.ts | 2 +- .../lib/copilot/tool-executor/handler-map.ts | 199 ++++++++++++++++ .../tool-executor/register-handlers.ts | 223 ++---------------- .../lib/copilot/tool-executor/router.test.ts | 8 +- .../get-blocks-metadata-projection.test.ts | 7 + apps/sim/lib/oauth/oauth.test.ts | 3 + .../operations/export-workflow.test.ts | 7 + .../workflows/search-replace/indexer.test.ts | 9 +- .../search-replace/replacements.test.ts | 9 +- .../serializer/tests/dual-validation.test.ts | 7 + .../tools/azure_devops/azure-devops.test.ts | 5 +- ...ranch_protection_and_workflow_runs.test.ts | 9 +- apps/sim/tools/metadata.test.ts | 9 +- apps/sim/tools/netsuite/netsuite.test.ts | 7 +- .../tools/okta/user_path_descriptions.test.ts | 12 +- apps/sim/tools/pitchbook/pitchbook.test.ts | 12 +- apps/sim/tools/semrush/semrush.test.ts | 12 +- apps/sim/tools/windchill/registry.test.ts | 9 +- apps/sim/vitest.config.ts | 13 + apps/sim/vitest.setup.ts | 88 +++++-- package.json | 16 +- packages/testing/package.json | 4 + .../src/factories/tool-responses.factory.ts | 6 +- .../testing/src/mocks/tool-registry.mock.ts | 38 +++ scripts/check-script-test-coverage.ts | 71 +++--- turbo.json | 1 + vitest.scripts.config.ts | 18 ++ 36 files changed, 621 insertions(+), 329 deletions(-) create mode 100644 apps/sim/lib/copilot/tool-executor/handler-map.ts create mode 100644 packages/testing/src/mocks/tool-registry.mock.ts create mode 100644 vitest.scripts.config.ts diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 690905622d2..9421c157b24 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -180,6 +180,11 @@ jobs: # Runs the setup CLI's Bun tests plus each workspace's Vitest suite, # without `--coverage`. See the Codecov note below. + # + # apps/sim runs only its first shard here; `test-shard` below runs the + # other. That suite is bound by the single Vite server thread that feeds + # every worker — wall time is flat from 4 to 13 workers — so a bigger + # runner buys nothing and a second runner halves it. - name: Run tests env: NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' @@ -187,6 +192,7 @@ jobs: DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' ENCRYPTION_KEY: '7cf672e460e430c1fba707575c2b0e2ad5a99dddf9b7b7e3b5646e630861db1c' # dummy key for CI only TURBO_CACHE_DIR: .turbo + SIM_TEST_SHARD: 1/2 run: bun run test - name: Check schema and migrations are in sync @@ -202,6 +208,67 @@ jobs: fi echo "✅ Schema and migrations are in sync" + # The second half of apps/sim's Vitest suite. Everything else — lint, the + # audits, type-check, the other workspaces' suites — lives in `test-build` + # with shard 1; this job exists only because that suite cannot go faster on + # one machine (see the "Run tests" note there). The Turbo cache disk gets + # its own key so the two shards' entries do not evict each other. + test-shard: + name: Test (shard 2) + runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} + timeout-minutes: 15 + + steps: + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.3.14 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6 + with: + node-version: 24 + + - name: Mount Bun cache + uses: ./.github/actions/cache-mount + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ~/.bun/install/cache + + - name: Mount node_modules + uses: ./.github/actions/cache-mount + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}-${{ hashFiles('bun.lock') }} + path: ./node_modules + + - name: Mount Turbo cache + uses: ./.github/actions/cache-mount + with: + provider: ${{ vars.CI_PROVIDER }} + key: ${{ github.repository }}-turbo-cache-shard-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }} + path: ./.turbo + + - name: Install dependencies + run: bun install --frozen-lockfile --ignore-scripts + + - name: Install ripgrep + run: command -v rg || (sudo apt-get update && sudo apt-get install -y ripgrep) + + - name: Run tests (apps/sim shard 2/2) + env: + NODE_OPTIONS: '--no-warnings --max-old-space-size=8192' + NEXT_PUBLIC_APP_URL: 'https://www.sim.ai' + DATABASE_URL: 'postgresql://postgres:postgres@localhost:5432/simstudio' + ENCRYPTION_KEY: '0000000000000000000000000000000000000000000000000000000000000000' # dummy key for CI only + TURBO_CACHE_DIR: .turbo + SIM_TEST_SHARD: 2/2 + run: bunx turbo run test --filter=@sim/app + # Next.js production build, in parallel with lint + tests. Sticky disks are # cloned from the last committed snapshot per job and committed last-writer- # wins, so concurrent mounts are safe. The bun/node_modules disks are shared diff --git a/apps/sim/app/api/copilot/tools/execute/route.ts b/apps/sim/app/api/copilot/tools/execute/route.ts index f645a5cb522..d247d93ee99 100644 --- a/apps/sim/app/api/copilot/tools/execute/route.ts +++ b/apps/sim/app/api/copilot/tools/execute/route.ts @@ -162,7 +162,7 @@ export const POST = withRouteHandler((request: NextRequest) => // glob/read/grep, function execute, ...) plus the server tool router // fallback — the plain server-tool adapter alone rejects VFS tools // with "Unknown server tool". - ensureHandlersRegistered() + await ensureHandlersRegistered() const result = await executeTool(toolName, params, { userId, workflowId: workflowId ?? '', diff --git a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-icon.test.ts b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-icon.test.ts index 1b8960913ee..432347418a5 100644 --- a/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-icon.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/rich-markdown-editor/mention/mention-icon.test.ts @@ -1,11 +1,14 @@ /** @vitest-environment node */ import { Workflow } from '@sim/emcn/icons' -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { AgentSkillsIcon } from '@/components/icons' import { getDocumentIcon } from '@/components/icons/document-icons' import { mentionIcon } from './mention-icon' import type { MentionKind } from './types' +/** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */ +vi.unmock('@/components/icons') + describe('mentionIcon', () => { it('uses the product-wide glyph for a known kind', () => { expect(mentionIcon('workflow', 'x')).toBe(Workflow) diff --git a/apps/sim/blocks/blocks/harmonic.test.ts b/apps/sim/blocks/blocks/harmonic.test.ts index e6a670cc7f2..6293b7f3677 100644 --- a/apps/sim/blocks/blocks/harmonic.test.ts +++ b/apps/sim/blocks/blocks/harmonic.test.ts @@ -3,11 +3,19 @@ */ import { describe, expect, it, vi } from 'vitest' -vi.unmock('@/tools/registry') +/** + * Only this service's configs are needed; the full registry is ~6,000 modules. + * Registration is asserted through the generated `@/tools/tool-ids`. + */ +vi.mock('@/tools/registry', async () => { + const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock') + return { tools: partialToolRegistry(await import('@/tools/harmonic')) } +}) import { HarmonicBlock, HarmonicBlockMeta } from '@/blocks/blocks/harmonic' import { BLOCK_META_REGISTRY, BLOCK_REGISTRY } from '@/blocks/registry-maps' import { tools } from '@/tools/registry' +import { hasToolId } from '@/tools/tool-ids' describe('HarmonicBlock', () => { const buildParams = HarmonicBlock.tools.config!.params! @@ -44,7 +52,8 @@ describe('HarmonicBlock', () => { for (const operation of operationIds) { const tool = tools[operation] - expect(tool?.id, `missing registry entry ${operation}`).toBe(operation) + expect(hasToolId(operation), `missing registry entry ${operation}`).toBe(true) + expect(tool?.id).toBe(operation) const blockOutputs = Object.entries(HarmonicBlock.outputs) .filter(([, output]) => { diff --git a/apps/sim/blocks/blocks/outlook.test.ts b/apps/sim/blocks/blocks/outlook.test.ts index c8b36c5ad46..69b3e68012f 100644 --- a/apps/sim/blocks/blocks/outlook.test.ts +++ b/apps/sim/blocks/blocks/outlook.test.ts @@ -1,15 +1,18 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { tools as toolRegistry } from '@/tools/registry' import { OutlookBlock } from './outlook' /** - * Uses the real tool registry: these assertions are about tool registration and - * params, which the global `@/tools/registry` mock in vitest.setup.ts empties. + * Only this service's configs are needed; the full registry is ~6,000 modules. + * Registration is asserted through the generated `@/tools/tool-ids`. */ -vi.unmock('@/tools/registry') +vi.mock('@/tools/registry', async () => { + const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock') + return { tools: partialToolRegistry(await import('@/tools/outlook')) } +}) const block = OutlookBlock diff --git a/apps/sim/blocks/brand-icon.test.tsx b/apps/sim/blocks/brand-icon.test.tsx index 75e62f7a95e..c1517658e7f 100644 --- a/apps/sim/blocks/brand-icon.test.tsx +++ b/apps/sim/blocks/brand-icon.test.tsx @@ -8,6 +8,9 @@ import { OAUTH_PROVIDERS } from '@/lib/oauth' import { BrandIcon, withBrandIcon } from '@/blocks/brand-icon' import { getAllBlocks } from '@/blocks/registry' +/** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */ +vi.unmock('@/components/icons') + vi.mocked(getAllBlocks).mockReturnValue([ { icon: DropboxIcon, iconColor: '#0061FF' }, ] as unknown as ReturnType) diff --git a/apps/sim/lib/billing/core/limit-notifications.ts b/apps/sim/lib/billing/core/limit-notifications.ts index 2df581f402a..698bcf60239 100644 --- a/apps/sim/lib/billing/core/limit-notifications.ts +++ b/apps/sim/lib/billing/core/limit-notifications.ts @@ -3,7 +3,6 @@ import { member, organization, settings, user, userStats } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { isOrgAdminRole } from '@sim/platform-authz/workspace' import { and, eq, sql } from 'drizzle-orm' -import { getLimitEmailSubject, renderLimitThresholdEmail } from '@/components/emails' import type { HighestPrioritySubscription } from '@/lib/billing/core/plan' import { getHighestPrioritySubscription } from '@/lib/billing/core/subscription' import type { BillingEntity } from '@/lib/billing/core/usage-log' @@ -11,7 +10,6 @@ import { isOrgScopedSubscription } from '@/lib/billing/subscriptions/utils' import { buildUpgradeHref, type UpgradeReason } from '@/lib/billing/upgrade-reasons' import { isBillingEnabled } from '@/lib/core/config/env-flags' import { getBaseUrl } from '@/lib/core/utils/urls' -import { sendEmail } from '@/lib/messaging/email/mailer' import { getEmailPreferences } from '@/lib/messaging/email/unsubscribe' const logger = createLogger('LimitNotifications') @@ -218,6 +216,11 @@ export async function maybeSendLimitThresholdEmail(params: { const percentUsed = Math.min(100, Math.round(percent)) const upgradeLink = `${getBaseUrl()}${buildUpgradeHref(params.workspaceId, category)}` + const [{ getLimitEmailSubject, renderLimitThresholdEmail }, { sendEmail }] = await Promise.all([ + import('@/components/emails'), + import('@/lib/messaging/email/mailer'), + ]) + let sent = 0 for (const r of recipients) { try { diff --git a/apps/sim/lib/billing/core/usage.ts b/apps/sim/lib/billing/core/usage.ts index b0c5045af01..33daa66d0cb 100644 --- a/apps/sim/lib/billing/core/usage.ts +++ b/apps/sim/lib/billing/core/usage.ts @@ -4,14 +4,6 @@ import { createLogger } from '@sim/logger' import { isOrgAdminRole } from '@sim/platform-authz/workspace' import { generateId } from '@sim/utils/id' import { and, eq, isNull, sql } from 'drizzle-orm' -import { - getEmailSubject, - getLimitEmailSubject, - renderCreditsExhaustedEmail, - renderFreeTierUpgradeEmail, - renderUsageLimitReachedEmail, - renderUsageThresholdEmail, -} from '@/components/emails' import { getEffectiveBillingStatus } from '@/lib/billing/core/access' import { defaultBillingPeriod } from '@/lib/billing/core/billing-period' import { @@ -45,11 +37,23 @@ import { Decimal, toDecimal, toNumber } from '@/lib/billing/utils/decimal' import { isBillingEnabled } from '@/lib/core/config/env-flags' import { getBaseUrl } from '@/lib/core/utils/urls' import type { DbClient } from '@/lib/db/types' -import { sendEmail } from '@/lib/messaging/email/mailer' import { getEmailPreferences } from '@/lib/messaging/email/unsubscribe' const logger = createLogger('UsageManagement') +/** + * Email rendering pulls the React templates and every mail provider into the + * module graph, which is ~1.2s of imports on every route that reaches billing + * attribution. Load it only when a threshold email is actually being sent. + */ +async function loadEmailDelivery() { + const [emails, mailer] = await Promise.all([ + import('@/components/emails'), + import('@/lib/messaging/email/mailer'), + ]) + return { ...emails, sendEmail: mailer.sendEmail } +} + export interface OrgUsageLimitResult { limit: number minimum: number @@ -773,6 +777,7 @@ export async function maybeSendUsageThresholdEmail(params: { const prefs = await getEmailPreferences(email) if (prefs?.unsubscribeAll || prefs?.unsubscribeNotifications) return + const { renderUsageThresholdEmail, getEmailSubject, sendEmail } = await loadEmailDelivery() const html = await renderUsageThresholdEmail({ userName: name, planName: params.planName, @@ -798,6 +803,7 @@ export async function maybeSendUsageThresholdEmail(params: { const prefs = await getEmailPreferences(email) if (prefs?.unsubscribeAll || prefs?.unsubscribeNotifications) return + const { renderFreeTierUpgradeEmail, getEmailSubject, sendEmail } = await loadEmailDelivery() const html = await renderFreeTierUpgradeEmail({ userName: name, percentUsed: Math.min(100, Math.round(params.percentAfter)), @@ -830,6 +836,13 @@ export async function maybeSendUsageThresholdEmail(params: { const prefs = await getEmailPreferences(email) if (prefs?.unsubscribeAll || prefs?.unsubscribeNotifications) return + const { + renderCreditsExhaustedEmail, + renderUsageLimitReachedEmail, + getEmailSubject, + getLimitEmailSubject, + sendEmail, + } = await loadEmailDelivery() const html = useFreeCopy ? await renderCreditsExhaustedEmail({ userName: name, diff --git a/apps/sim/lib/catalog/projection/catalog-sweep.test.ts b/apps/sim/lib/catalog/projection/catalog-sweep.test.ts index dbc538ed6b1..d2677993179 100644 --- a/apps/sim/lib/catalog/projection/catalog-sweep.test.ts +++ b/apps/sim/lib/catalog/projection/catalog-sweep.test.ts @@ -41,6 +41,13 @@ import { getBlockRegistry } from '@/blocks/registry' import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' import { getToolIds } from '@/tools/tool-ids' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + /** Hosted deployment: the state under which every declared hosted key is published. */ const HOSTED: CatalogDeployment = { hostedKeys: true } diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index 158cd08897e..dc39782d37d 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -608,7 +608,7 @@ async function executeToolAndReportInner( } try { - ensureHandlersRegistered() + await ensureHandlersRegistered() let result = await executeToolWithWatchdog(toolCall, toolExecutionContext) if (toolCall.endTime || isTerminalToolCallStatus(toolCall.status)) { endToolSpanFromTerminalState() diff --git a/apps/sim/lib/copilot/tool-executor/handler-map.ts b/apps/sim/lib/copilot/tool-executor/handler-map.ts new file mode 100644 index 00000000000..f9b3b583a47 --- /dev/null +++ b/apps/sim/lib/copilot/tool-executor/handler-map.ts @@ -0,0 +1,199 @@ +import { + CancelWorkflowRun, + ConnectSlackBot, + Cp as CpTool, + CreateWorkflow, + CreateWorkspaceMcpServer, + DeleteWorkspaceMcpServer, + DeployAsApi, + DeployAsChat, + DeployAsMcp, + DiffWorkflows, + GenerateApiKey, + GetBlockOutputs, + GetBlockUpstreamReferences, + GetDeployedWorkflowState, + GetDeploymentStatus, + GetWorkflowData, + GetWorkflowRunOptions, + Glob as GlobTool, + Grep as GrepTool, + ListDeploymentVersions, + ListIntegrationTools, + ListWorkspaceMcpServers, + LoadDeployment, + ManageCredential, + ManageCustomTool, + ManageMcpConnection, + ManageSandbox, + ManageSkill, + Mkdir as MkdirTool, + Mv as MvTool, + OauthGetAuthLink, + OauthRequestAccess, + OpenResource, + PromoteToLive, + PublishCustomBlock, + Read as ReadTool, + Redeploy, + RestoreResource, + Rm as RmTool, + RunBlock, + RunCode, + RunFromBlock, + RunFunction, + RunWorkflow, + RunWorkflowUntilBlock, + SaveUpload, + SetBlockEnabled, + SetGlobalWorkflowVariables, + UpdateDeploymentVersion, + UpdateWorkspaceMcpServer, +} from '@/lib/copilot/generated/tool-catalog-v1' +import { createServerToolHandler } from '@/lib/copilot/tools/registry/server-tool-adapter' +import { getRegisteredServerToolNames } from '@/lib/copilot/tools/server/router' +import { executeDeployCustomBlock } from '../tools/handlers/deployment/custom-block' +import { + executeDeployApi, + executeDeployChat, + executeDeployMcp, + executeRedeploy, +} from '../tools/handlers/deployment/deploy' +import { + executeCheckDeploymentStatus, + executeCreateWorkspaceMcpServer, + executeDeleteWorkspaceMcpServer, + executeDiffWorkflows, + executeGetDeploymentLog, + executeListWorkspaceMcpServers, + executeLoadDeployment, + executePromoteToLive, + executeUpdateDeploymentVersion, + executeUpdateWorkspaceMcpServer, +} from '../tools/handlers/deployment/manage' +import { executeFunctionExecute } from '../tools/handlers/function-execute' +import { executeListIntegrationTools } from '../tools/handlers/integration-tools' +import { executeConnectSlackBot } from '../tools/handlers/management/connect-slack-bot' +import { executeManageCredential } from '../tools/handlers/management/manage-credential' +import { executeManageCustomTool } from '../tools/handlers/management/manage-custom-tool' +import { executeManageMcpTool } from '../tools/handlers/management/manage-mcp-tool' +import { executeManageSandbox } from '../tools/handlers/management/manage-sandbox' +import { executeManageSkill } from '../tools/handlers/management/manage-skill' +import { executeMaterializeFile } from '../tools/handlers/materialize-file' +import { executeOAuthGetAuthLink, executeOAuthRequestAccess } from '../tools/handlers/oauth' +import { executeOpenResource } from '../tools/handlers/resources' +import { executeRestoreResource } from '../tools/handlers/restore-resource' +import { executeRunCode } from '../tools/handlers/run-code' +import { executeVfsGlob, executeVfsGrep, executeVfsRead } from '../tools/handlers/vfs' +import { + executeVfsCp, + executeVfsMkdir, + executeVfsMv, + executeVfsRm, +} from '../tools/handlers/vfs-mutate' +import { + executeCancelWorkflowRun, + executeCreateWorkflow, + executeGenerateApiKey, + executeMoveWorkflow, + executeRenameWorkflow, + executeRunBlock, + executeRunFromBlock, + executeRunWorkflow, + executeRunWorkflowUntilBlock, + executeSetBlockEnabled, + executeSetGlobalWorkflowVariables, +} from '../tools/handlers/workflow/mutations' +import { + executeGetBlockOutputs, + executeGetBlockUpstreamReferences, + executeGetDeployedWorkflowState, + executeGetWorkflowData, + executeGetWorkflowRunOptions, +} from '../tools/handlers/workflow/queries' +import type { ToolHandler } from './types' + +// Bridge: handler implementations accept specific param types (e.g. CreateWorkflowParams) +// while ToolHandler accepts Record. The params are cast internally by +// each implementation. ExecutionContext extends ToolExecutionContext so context is compatible. +function h(fn: (params: any, context: any) => Promise): ToolHandler { + return fn as ToolHandler +} + +export function buildHandlerMap(): Record { + return { + [GetWorkflowData.id]: h(executeGetWorkflowData), + [GetWorkflowRunOptions.id]: h(executeGetWorkflowRunOptions), + [GetBlockOutputs.id]: h(executeGetBlockOutputs), + [GetBlockUpstreamReferences.id]: h(executeGetBlockUpstreamReferences), + [GetDeployedWorkflowState.id]: h(executeGetDeployedWorkflowState), + + [CreateWorkflow.id]: h(executeCreateWorkflow), + // rename_workflow / move_workflow were removed from the mothership catalog + // in favor of mv; the executors stay registered under literal names so + // in-flight checkpoints still resume. Delete after the mv release soaks. + rename_workflow: h(executeRenameWorkflow), + move_workflow: h(executeMoveWorkflow), + [RunWorkflow.id]: h(executeRunWorkflow), + [CancelWorkflowRun.id]: h(executeCancelWorkflowRun), + [RunWorkflowUntilBlock.id]: h(executeRunWorkflowUntilBlock), + [RunFromBlock.id]: h(executeRunFromBlock), + [RunBlock.id]: h(executeRunBlock), + [SetBlockEnabled.id]: h(executeSetBlockEnabled), + [GenerateApiKey.id]: h(executeGenerateApiKey), + [SetGlobalWorkflowVariables.id]: h(executeSetGlobalWorkflowVariables), + + [DeployAsApi.id]: h(executeDeployApi), + [DeployAsChat.id]: h(executeDeployChat), + [DeployAsMcp.id]: h(executeDeployMcp), + [PublishCustomBlock.id]: h(executeDeployCustomBlock), + [Redeploy.id]: h(executeRedeploy), + [GetDeploymentStatus.id]: h(executeCheckDeploymentStatus), + [ListWorkspaceMcpServers.id]: h(executeListWorkspaceMcpServers), + [CreateWorkspaceMcpServer.id]: h(executeCreateWorkspaceMcpServer), + [UpdateWorkspaceMcpServer.id]: h(executeUpdateWorkspaceMcpServer), + [DeleteWorkspaceMcpServer.id]: h(executeDeleteWorkspaceMcpServer), + [ListDeploymentVersions.id]: h(executeGetDeploymentLog), + [DiffWorkflows.id]: h(executeDiffWorkflows), + [LoadDeployment.id]: h(executeLoadDeployment), + [PromoteToLive.id]: h(executePromoteToLive), + [UpdateDeploymentVersion.id]: h(executeUpdateDeploymentVersion), + + [GrepTool.id]: h(executeVfsGrep), + [GlobTool.id]: h(executeVfsGlob), + [ReadTool.id]: h(executeVfsRead), + [MvTool.id]: h(executeVfsMv), + [CpTool.id]: h(executeVfsCp), + [MkdirTool.id]: h(executeVfsMkdir), + [RmTool.id]: h(executeVfsRm), + + [ManageCustomTool.id]: h(executeManageCustomTool), + [ManageMcpConnection.id]: h(executeManageMcpTool), + [ManageSandbox.id]: h(executeManageSandbox), + [ManageSkill.id]: h(executeManageSkill), + [ManageCredential.id]: h(executeManageCredential), + [ConnectSlackBot.id]: h(executeConnectSlackBot), + [OauthGetAuthLink.id]: h(executeOAuthGetAuthLink), + // Rolling-deploy compatibility for calls/checkpoints created before OAuth + // moved into terminal credential cards. New agents no longer receive this + // tool, but old calls must remain resumable until both services have soaked. + [OauthRequestAccess.id]: h(executeOAuthRequestAccess), + [OpenResource.id]: h(executeOpenResource), + [RestoreResource.id]: h(executeRestoreResource), + [ListIntegrationTools.id]: h(executeListIntegrationTools), + [SaveUpload.id]: h(executeMaterializeFile), + [RunFunction.id]: h(executeFunctionExecute), + [RunCode.id]: h(executeRunCode), + + ...buildServerToolHandlers(), + } +} + +function buildServerToolHandlers(): Record { + const toolNames = getRegisteredServerToolNames() + const handlers: Record = {} + for (const toolId of toolNames) { + handlers[toolId] = createServerToolHandler(toolId) + } + return handlers +} diff --git a/apps/sim/lib/copilot/tool-executor/register-handlers.ts b/apps/sim/lib/copilot/tool-executor/register-handlers.ts index 5655d7aa2ba..e77644d08fe 100644 --- a/apps/sim/lib/copilot/tool-executor/register-handlers.ts +++ b/apps/sim/lib/copilot/tool-executor/register-handlers.ts @@ -1,212 +1,23 @@ import { createLogger } from '@sim/logger' -import { - CancelWorkflowRun, - ConnectSlackBot, - Cp as CpTool, - CreateWorkflow, - CreateWorkspaceMcpServer, - DeleteWorkspaceMcpServer, - DeployAsApi, - DeployAsChat, - DeployAsMcp, - DiffWorkflows, - GenerateApiKey, - GetBlockOutputs, - GetBlockUpstreamReferences, - GetDeployedWorkflowState, - GetDeploymentStatus, - GetWorkflowData, - GetWorkflowRunOptions, - Glob as GlobTool, - Grep as GrepTool, - ListDeploymentVersions, - ListIntegrationTools, - ListWorkspaceMcpServers, - LoadDeployment, - ManageCredential, - ManageCustomTool, - ManageMcpConnection, - ManageSandbox, - ManageSkill, - Mkdir as MkdirTool, - Mv as MvTool, - OauthGetAuthLink, - OauthRequestAccess, - OpenResource, - PromoteToLive, - PublishCustomBlock, - Read as ReadTool, - Redeploy, - RestoreResource, - Rm as RmTool, - RunBlock, - RunCode, - RunFromBlock, - RunFunction, - RunWorkflow, - RunWorkflowUntilBlock, - SaveUpload, - SetBlockEnabled, - SetGlobalWorkflowVariables, - UpdateDeploymentVersion, - UpdateWorkspaceMcpServer, -} from '@/lib/copilot/generated/tool-catalog-v1' -import { createServerToolHandler } from '@/lib/copilot/tools/registry/server-tool-adapter' -import { getRegisteredServerToolNames } from '@/lib/copilot/tools/server/router' -import { executeDeployCustomBlock } from '../tools/handlers/deployment/custom-block' -import { - executeDeployApi, - executeDeployChat, - executeDeployMcp, - executeRedeploy, -} from '../tools/handlers/deployment/deploy' -import { - executeCheckDeploymentStatus, - executeCreateWorkspaceMcpServer, - executeDeleteWorkspaceMcpServer, - executeDiffWorkflows, - executeGetDeploymentLog, - executeListWorkspaceMcpServers, - executeLoadDeployment, - executePromoteToLive, - executeUpdateDeploymentVersion, - executeUpdateWorkspaceMcpServer, -} from '../tools/handlers/deployment/manage' -import { executeFunctionExecute } from '../tools/handlers/function-execute' -import { executeListIntegrationTools } from '../tools/handlers/integration-tools' -import { executeConnectSlackBot } from '../tools/handlers/management/connect-slack-bot' -import { executeManageCredential } from '../tools/handlers/management/manage-credential' -import { executeManageCustomTool } from '../tools/handlers/management/manage-custom-tool' -import { executeManageMcpTool } from '../tools/handlers/management/manage-mcp-tool' -import { executeManageSandbox } from '../tools/handlers/management/manage-sandbox' -import { executeManageSkill } from '../tools/handlers/management/manage-skill' -import { executeMaterializeFile } from '../tools/handlers/materialize-file' -import { executeOAuthGetAuthLink, executeOAuthRequestAccess } from '../tools/handlers/oauth' -import { executeOpenResource } from '../tools/handlers/resources' -import { executeRestoreResource } from '../tools/handlers/restore-resource' -import { executeRunCode } from '../tools/handlers/run-code' -import { executeVfsGlob, executeVfsGrep, executeVfsRead } from '../tools/handlers/vfs' -import { - executeVfsCp, - executeVfsMkdir, - executeVfsMv, - executeVfsRm, -} from '../tools/handlers/vfs-mutate' -import { - executeCancelWorkflowRun, - executeCreateWorkflow, - executeGenerateApiKey, - executeMoveWorkflow, - executeRenameWorkflow, - executeRunBlock, - executeRunFromBlock, - executeRunWorkflow, - executeRunWorkflowUntilBlock, - executeSetBlockEnabled, - executeSetGlobalWorkflowVariables, -} from '../tools/handlers/workflow/mutations' -import { - executeGetBlockOutputs, - executeGetBlockUpstreamReferences, - executeGetDeployedWorkflowState, - executeGetWorkflowData, - executeGetWorkflowRunOptions, -} from '../tools/handlers/workflow/queries' import { registerHandlers } from './executor' -import type { ToolHandler } from './types' const logger = createLogger('ToolHandlerRegistration') -let registered = false - -export function ensureHandlersRegistered(): void { - if (registered) return - registered = true - registerHandlers(buildHandlerMap()) - logger.info('Tool handlers registered') -} - -// Bridge: handler implementations accept specific param types (e.g. CreateWorkflowParams) -// while ToolHandler accepts Record. The params are cast internally by -// each implementation. ExecutionContext extends ToolExecutionContext so context is compatible. -function h(fn: (params: any, context: any) => Promise): ToolHandler { - return fn as ToolHandler -} - -function buildHandlerMap(): Record { - return { - [GetWorkflowData.id]: h(executeGetWorkflowData), - [GetWorkflowRunOptions.id]: h(executeGetWorkflowRunOptions), - [GetBlockOutputs.id]: h(executeGetBlockOutputs), - [GetBlockUpstreamReferences.id]: h(executeGetBlockUpstreamReferences), - [GetDeployedWorkflowState.id]: h(executeGetDeployedWorkflowState), - - [CreateWorkflow.id]: h(executeCreateWorkflow), - // rename_workflow / move_workflow were removed from the mothership catalog - // in favor of mv; the executors stay registered under literal names so - // in-flight checkpoints still resume. Delete after the mv release soaks. - rename_workflow: h(executeRenameWorkflow), - move_workflow: h(executeMoveWorkflow), - [RunWorkflow.id]: h(executeRunWorkflow), - [CancelWorkflowRun.id]: h(executeCancelWorkflowRun), - [RunWorkflowUntilBlock.id]: h(executeRunWorkflowUntilBlock), - [RunFromBlock.id]: h(executeRunFromBlock), - [RunBlock.id]: h(executeRunBlock), - [SetBlockEnabled.id]: h(executeSetBlockEnabled), - [GenerateApiKey.id]: h(executeGenerateApiKey), - [SetGlobalWorkflowVariables.id]: h(executeSetGlobalWorkflowVariables), - - [DeployAsApi.id]: h(executeDeployApi), - [DeployAsChat.id]: h(executeDeployChat), - [DeployAsMcp.id]: h(executeDeployMcp), - [PublishCustomBlock.id]: h(executeDeployCustomBlock), - [Redeploy.id]: h(executeRedeploy), - [GetDeploymentStatus.id]: h(executeCheckDeploymentStatus), - [ListWorkspaceMcpServers.id]: h(executeListWorkspaceMcpServers), - [CreateWorkspaceMcpServer.id]: h(executeCreateWorkspaceMcpServer), - [UpdateWorkspaceMcpServer.id]: h(executeUpdateWorkspaceMcpServer), - [DeleteWorkspaceMcpServer.id]: h(executeDeleteWorkspaceMcpServer), - [ListDeploymentVersions.id]: h(executeGetDeploymentLog), - [DiffWorkflows.id]: h(executeDiffWorkflows), - [LoadDeployment.id]: h(executeLoadDeployment), - [PromoteToLive.id]: h(executePromoteToLive), - [UpdateDeploymentVersion.id]: h(executeUpdateDeploymentVersion), - - [GrepTool.id]: h(executeVfsGrep), - [GlobTool.id]: h(executeVfsGlob), - [ReadTool.id]: h(executeVfsRead), - [MvTool.id]: h(executeVfsMv), - [CpTool.id]: h(executeVfsCp), - [MkdirTool.id]: h(executeVfsMkdir), - [RmTool.id]: h(executeVfsRm), - - [ManageCustomTool.id]: h(executeManageCustomTool), - [ManageMcpConnection.id]: h(executeManageMcpTool), - [ManageSandbox.id]: h(executeManageSandbox), - [ManageSkill.id]: h(executeManageSkill), - [ManageCredential.id]: h(executeManageCredential), - [ConnectSlackBot.id]: h(executeConnectSlackBot), - [OauthGetAuthLink.id]: h(executeOAuthGetAuthLink), - // Rolling-deploy compatibility for calls/checkpoints created before OAuth - // moved into terminal credential cards. New agents no longer receive this - // tool, but old calls must remain resumable until both services have soaked. - [OauthRequestAccess.id]: h(executeOAuthRequestAccess), - [OpenResource.id]: h(executeOpenResource), - [RestoreResource.id]: h(executeRestoreResource), - [ListIntegrationTools.id]: h(executeListIntegrationTools), - [SaveUpload.id]: h(executeMaterializeFile), - [RunFunction.id]: h(executeFunctionExecute), - [RunCode.id]: h(executeRunCode), - - ...buildServerToolHandlers(), - } -} - -function buildServerToolHandlers(): Record { - const toolNames = getRegisteredServerToolNames() - const handlers: Record = {} - for (const toolId of toolNames) { - handlers[toolId] = createServerToolHandler(toolId) - } - return handlers +let registration: Promise | null = null + +/** + * Registers every server-side tool handler exactly once. + * + * The handler map statically imports every copilot tool implementation, which + * transitively reaches the block registry, table application layer, and most + * of `lib/` — several thousand modules. Nothing that merely routes or inspects + * tool calls needs any of that, so the map is loaded on first execution rather + * than whenever this module is imported. + */ +export function ensureHandlersRegistered(): Promise { + registration ??= import('./handler-map').then(({ buildHandlerMap }) => { + registerHandlers(buildHandlerMap()) + logger.info('Tool handlers registered') + }) + return registration } diff --git a/apps/sim/lib/copilot/tool-executor/router.test.ts b/apps/sim/lib/copilot/tool-executor/router.test.ts index f031cfc18b5..700ec190b6f 100644 --- a/apps/sim/lib/copilot/tool-executor/router.test.ts +++ b/apps/sim/lib/copilot/tool-executor/router.test.ts @@ -21,9 +21,11 @@ describe('workflow-run cancellation tool routing', () => { expect(toolRequiresApproval('cancel_workflow_run')).toBe(true) }) - it('registers the Sim cancellation handler', () => { - ensureHandlersRegistered() + // Registration loads the whole handler map on first use, which is most of + // `lib/` — well past the default 10s under a fully parallel run. + it('registers the Sim cancellation handler', async () => { + await ensureHandlersRegistered() expect(hasHandler('cancel_workflow_run')).toBe(true) - }) + }, 90_000) }) diff --git a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-projection.test.ts b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-projection.test.ts index e0e2a5688ca..a99a8090b63 100644 --- a/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-projection.test.ts +++ b/apps/sim/lib/copilot/tools/server/blocks/get-blocks-metadata-projection.test.ts @@ -30,6 +30,13 @@ vi.mock('@/lib/integrations/availability.server', () => ({ import { getBlocksMetadataServerTool } from '@/lib/copilot/tools/server/blocks/get-blocks-metadata-tool' +/** + * The projection under test reads real tool params and outputs, which the global + * `@/tools/metadata` and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + interface AgentBlockMetadata { blockType: string name: string diff --git a/apps/sim/lib/oauth/oauth.test.ts b/apps/sim/lib/oauth/oauth.test.ts index f1f39d28504..3aadd1da130 100644 --- a/apps/sim/lib/oauth/oauth.test.ts +++ b/apps/sim/lib/oauth/oauth.test.ts @@ -78,6 +78,9 @@ import { } from '@/lib/oauth' import { REDDIT_USER_AGENT } from '@/tools/reddit/constants' +/** Compares real icon components by identity; the global `@/components/icons` stub in vitest.setup.ts would make that vacuous. */ +vi.unmock('@/components/icons') + /** * Default OAuth token response for successful requests. */ diff --git a/apps/sim/lib/workflows/operations/export-workflow.test.ts b/apps/sim/lib/workflows/operations/export-workflow.test.ts index 18ca2c7abff..07a32708768 100644 --- a/apps/sim/lib/workflows/operations/export-workflow.test.ts +++ b/apps/sim/lib/workflows/operations/export-workflow.test.ts @@ -32,6 +32,13 @@ vi.mock('@/blocks/registry', () => ({ import { buildWorkflowExportPayload } from '@/lib/workflows/operations/export-workflow' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + describe('buildWorkflowExportPayload', () => { beforeEach(() => { vi.clearAllMocks() diff --git a/apps/sim/lib/workflows/search-replace/indexer.test.ts b/apps/sim/lib/workflows/search-replace/indexer.test.ts index 099e99f3036..bc396bf4024 100644 --- a/apps/sim/lib/workflows/search-replace/indexer.test.ts +++ b/apps/sim/lib/workflows/search-replace/indexer.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { getToolInputParamConfigs, indexWorkflowSearchMatches, @@ -14,6 +14,13 @@ import { import { WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS } from '@/lib/workflows/search-replace/subflow-fields' import { NoteBlock } from '@/blocks/blocks/note' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + /** * Uses the real tool registry. Nothing here imports it directly — the dependency * is transitive: the search-replace planner resolves tool input params through diff --git a/apps/sim/lib/workflows/search-replace/replacements.test.ts b/apps/sim/lib/workflows/search-replace/replacements.test.ts index 340fa434350..224ff18c628 100644 --- a/apps/sim/lib/workflows/search-replace/replacements.test.ts +++ b/apps/sim/lib/workflows/search-replace/replacements.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { indexWorkflowSearchMatches } from '@/lib/workflows/search-replace/indexer' import { buildWorkflowSearchReplacePlan } from '@/lib/workflows/search-replace/replacements' import { @@ -10,6 +10,13 @@ import { } from '@/lib/workflows/search-replace/search-replace.fixtures' import { WORKFLOW_SEARCH_SUBFLOW_FIELD_IDS } from '@/lib/workflows/search-replace/subflow-fields' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + /** * Uses the real tool registry. Nothing here imports it directly — the dependency * is transitive: the search-replace planner resolves tool input params through diff --git a/apps/sim/serializer/tests/dual-validation.test.ts b/apps/sim/serializer/tests/dual-validation.test.ts index 4704040e0af..7d7ec7384f8 100644 --- a/apps/sim/serializer/tests/dual-validation.test.ts +++ b/apps/sim/serializer/tests/dual-validation.test.ts @@ -12,6 +12,13 @@ import { afterAll, describe, expect, it, vi } from 'vitest' import * as subblockVisibility from '@/lib/workflows/subblocks/visibility' import { Serializer } from '@/serializer/index' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + vi.mock('@/blocks', () => blocksMock) /** diff --git a/apps/sim/tools/azure_devops/azure-devops.test.ts b/apps/sim/tools/azure_devops/azure-devops.test.ts index 520b143675e..b1c126f22fb 100644 --- a/apps/sim/tools/azure_devops/azure-devops.test.ts +++ b/apps/sim/tools/azure_devops/azure-devops.test.ts @@ -2,8 +2,8 @@ * @vitest-environment node */ import { afterEach, describe, expect, it, vi } from 'vitest' +import { hasToolId } from '@/tools/tool-ids' import { isAzureDevOpsEventMatch } from '@/triggers/azure_devops/utils' -import { tools } from '../registry' import type { ToolConfig } from '../types' import { addCommentTool } from './add_comment' import { createWorkItemTool } from './create_work_item' @@ -49,7 +49,6 @@ const baseParams = { * Uses the real tool registry: these assertions are about tool registration and * params, which the global `@/tools/registry` mock in vitest.setup.ts empties. */ -vi.unmock('@/tools/registry') const authHeader = `Basic ${Buffer.from(':pat-token').toString('base64')}` @@ -136,7 +135,7 @@ describe('Azure DevOps tool contracts', () => { expect(allTools.map((tool) => tool.id).sort()).toEqual(expectedIds) for (const id of expectedIds) { - expect(tools[id]?.id).toBe(id) + expect(hasToolId(id), id).toBe(true) } }) diff --git a/apps/sim/tools/github/branch_protection_and_workflow_runs.test.ts b/apps/sim/tools/github/branch_protection_and_workflow_runs.test.ts index 1d7ad125d4f..f309353c738 100644 --- a/apps/sim/tools/github/branch_protection_and_workflow_runs.test.ts +++ b/apps/sim/tools/github/branch_protection_and_workflow_runs.test.ts @@ -10,7 +10,14 @@ import { } from '@/tools/github/update_branch_protection' import { getTool, validateRequiredParametersAfterMerge } from '@/tools/utils' -vi.unmock('@/tools/registry') +/** + * Only this service's configs are needed; the full registry is ~6,000 modules. + * Registration is asserted through the generated `@/tools/tool-ids`. + */ +vi.mock('@/tools/registry', async () => { + const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock') + return { tools: partialToolRegistry(await import('@/tools/github')) } +}) const BASE_PROTECTION_PARAMS = { owner: 'sim', diff --git a/apps/sim/tools/metadata.test.ts b/apps/sim/tools/metadata.test.ts index 2b0f49b2412..4f57faec179 100644 --- a/apps/sim/tools/metadata.test.ts +++ b/apps/sim/tools/metadata.test.ts @@ -1,11 +1,18 @@ /** * @vitest-environment node */ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' import { getToolMetadata, getToolParams } from '@/tools/metadata' import { getToolOutputsMetadata } from '@/tools/metadata-outputs' import { getToolIds, hasToolId, resolveToolId } from '@/tools/tool-ids' +/** + * Asserts real tool params and outputs, which the global `@/tools/metadata` + * and `@/tools/metadata-outputs` mocks in vitest.setup.ts empty. + */ +vi.unmock('@/tools/metadata') +vi.unmock('@/tools/metadata-outputs') + /** * Guards the properties the generated artifacts are relied on for. The * `tool-metadata:check` script guards that they are in sync with the registry; diff --git a/apps/sim/tools/netsuite/netsuite.test.ts b/apps/sim/tools/netsuite/netsuite.test.ts index 4f9e7c048d6..7092e862858 100644 --- a/apps/sim/tools/netsuite/netsuite.test.ts +++ b/apps/sim/tools/netsuite/netsuite.test.ts @@ -4,9 +4,6 @@ import { readFileSync } from 'node:fs' import { resolve } from 'node:path' import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest' - -vi.unmock('@/tools/registry') - import { executeNetsuiteTool } from '@/lib/internal/netsuite/execute-tool' import { executeNetsuiteAttachRecordOperation } from '@/lib/internal/netsuite/operations/attach-record' import { executeNetsuiteGetSelectOptionsOperation } from '@/lib/internal/netsuite/operations/get-select-options' @@ -47,7 +44,7 @@ import { } from '@/tools/netsuite' import type { NetSuiteAuthParams } from '@/tools/netsuite/types' import { netsuiteAuthParamFields } from '@/tools/netsuite/utils' -import { tools } from '@/tools/registry' +import { hasToolId } from '@/tools/tool-ids' import type { InternalToolConfig, ToolResponse } from '@/tools/types' const ORIGIN = 'https://1234567.suitetalk.api.netsuite.com' @@ -875,7 +872,7 @@ describe('NetSuite operation contracts', () => { for (const id of matrixIds) { expect(NetSuiteBlock.tools.config.tool({ operation: id }), `${id} mapping`).toBe(id) - expect(tools[id]?.id, `${id} registry`).toBe(id) + expect(hasToolId(id), `${id} registry`).toBe(true) expect(toolMetadata[id]?.id, `${id} generated metadata`).toBe(id) } }) diff --git a/apps/sim/tools/okta/user_path_descriptions.test.ts b/apps/sim/tools/okta/user_path_descriptions.test.ts index fa8e62b4a97..089de014e36 100644 --- a/apps/sim/tools/okta/user_path_descriptions.test.ts +++ b/apps/sim/tools/okta/user_path_descriptions.test.ts @@ -1,16 +1,10 @@ /** * @vitest-environment node */ -import { describe, expect, it, vi } from 'vitest' -import { tools as toolRegistry } from '@/tools/registry' +import { describe, expect, it } from 'vitest' +import * as oktaTools from '@/tools/okta' import type { ToolConfig } from '@/tools/types' -/** - * Uses the real tool registry: these assertions are about registered Okta tool - * params, which the global `@/tools/registry` mock in vitest.setup.ts empties. - */ -vi.unmock('@/tools/registry') - /** * Okta's Management API spec (`okta/okta-management-openapi-spec`, * `dist/2026.08.1/management-oneOfInheritance-noExamples.yaml`) distinguishes @@ -68,7 +62,7 @@ function builtUrl(tool: ToolConfig): string { return build(sentinelParams(tool) as never) } -const oktaUserTools: OktaUserTool[] = Object.values(toolRegistry) +const oktaUserTools: OktaUserTool[] = Object.values(oktaTools) .filter((tool): tool is ToolConfig => Boolean(tool?.id?.startsWith('okta_'))) .filter((tool) => Boolean(tool.params?.userId)) .map((tool) => ({ diff --git a/apps/sim/tools/pitchbook/pitchbook.test.ts b/apps/sim/tools/pitchbook/pitchbook.test.ts index 330214aca55..b816a40f217 100644 --- a/apps/sim/tools/pitchbook/pitchbook.test.ts +++ b/apps/sim/tools/pitchbook/pitchbook.test.ts @@ -3,13 +3,21 @@ */ import { describe, expect, it, vi } from 'vitest' -vi.unmock('@/tools/registry') +/** + * Only this service's configs are needed; the full registry is ~6,000 modules. + * Registration is asserted through the generated `@/tools/tool-ids`. + */ +vi.mock('@/tools/registry', async () => { + const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock') + return { tools: partialToolRegistry(await import('@/tools/pitchbook')) } +}) import { PitchBookBlock } from '@/blocks/blocks/pitchbook' import { ErrorExtractorId, extractErrorMessage, redactErrorData } from '@/tools/error-extractors' import { executeTool } from '@/tools/index' import RECORDED from '@/tools/pitchbook/__fixtures__/recorded-responses.json' import { tools } from '@/tools/registry' +import { hasToolId } from '@/tools/tool-ids' /** * The registry is typed `Record`, so a tool's `url`/`headers`/ @@ -48,7 +56,7 @@ describe('pitchbook wiring', () => { it('registry keys match each tool id', () => { for (const id of access) { - expect(tools[id], `missing registry entry ${id}`).toBeDefined() + expect(hasToolId(id), `missing registry entry ${id}`).toBe(true) expect(tools[id].id).toBe(id) } }) diff --git a/apps/sim/tools/semrush/semrush.test.ts b/apps/sim/tools/semrush/semrush.test.ts index 79ef326ef49..6780893e5ca 100644 --- a/apps/sim/tools/semrush/semrush.test.ts +++ b/apps/sim/tools/semrush/semrush.test.ts @@ -6,8 +6,14 @@ */ import { describe, expect, it, vi } from 'vitest' -// The registry is globally mocked for import cost; this file asserts registration. -vi.unmock('@/tools/registry') +/** + * Only this service's configs are needed; the full registry is ~6,000 modules. + * Registration is asserted through the generated `@/tools/tool-ids`. + */ +vi.mock('@/tools/registry', async () => { + const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock') + return { tools: partialToolRegistry(await import('@/tools/semrush')) } +}) import { SemrushBlock } from '@/blocks/blocks/semrush' import type { SubBlockConfig } from '@/blocks/types' @@ -22,6 +28,7 @@ import { semrushOrganicResultsTool } from '@/tools/semrush/organic_results' import { semrushReferringDomainsTool } from '@/tools/semrush/referring_domains' import { getColumnDef } from '@/tools/semrush/utils' import { semrushWinnersAndLosersTool } from '@/tools/semrush/winners_and_losers' +import { hasToolId } from '@/tools/tool-ids' import type { ToolConfig } from '@/tools/types' function csvResponse(body: string, status = 200): Response { @@ -406,6 +413,7 @@ describe('semrush registry surface', () => { expect(semrushTools).toHaveLength(44) for (const [id, tool] of semrushTools) { expect((tool as ToolConfig).id).toBe(id) + expect(hasToolId(id), `${id} registry`).toBe(true) } }) diff --git a/apps/sim/tools/windchill/registry.test.ts b/apps/sim/tools/windchill/registry.test.ts index c800cbb7ff8..80086e13a5d 100644 --- a/apps/sim/tools/windchill/registry.test.ts +++ b/apps/sim/tools/windchill/registry.test.ts @@ -2,17 +2,14 @@ * @vitest-environment node */ -import { describe, expect, it, vi } from 'vitest' - -vi.unmock('@/tools/registry') - -import { tools } from '@/tools/registry' +import { describe, expect, it } from 'vitest' +import { hasToolId } from '@/tools/tool-ids' import { WINDCHILL_OPERATIONS } from '@/tools/windchill/types' describe('Windchill registry', () => { it('registers every operation in the global tool registry', () => { for (const operation of WINDCHILL_OPERATIONS) { - expect(tools[operation]?.id).toBe(operation) + expect(hasToolId(operation), operation).toBe(true) } }) }) diff --git a/apps/sim/vitest.config.ts b/apps/sim/vitest.config.ts index 6b62aacb575..56fa8b81ceb 100644 --- a/apps/sim/vitest.config.ts +++ b/apps/sim/vitest.config.ts @@ -11,7 +11,14 @@ loadEnvConfig(projectDir) export default defineConfig({ plugins: [react()], + /** + * Skip PostCSS entirely. Loading the Tailwind config for a `.module.css` + * import costs ~150ms per test file that reaches an emcn component, and no + * test reads real CSS. + */ + css: { postcss: {} }, test: { + css: false, globals: true, environment: 'node', include: ['**/*.test.{ts,tsx}'], @@ -24,6 +31,12 @@ export default defineConfig({ fileParallelism: true, maxConcurrency: 10, testTimeout: 10000, + /** + * CI splits this suite across runners (`1/2`, `2/2`). A single Vite server + * thread feeds every worker, so throughput stops scaling at ~4 workers on + * one machine; more machines is the only parallelism left. + */ + shard: process.env.SIM_TEST_SHARD || undefined, }, resolve: { tsconfigPaths: true, diff --git a/apps/sim/vitest.setup.ts b/apps/sim/vitest.setup.ts index 910025e6839..eb92ca8d8c6 100644 --- a/apps/sim/vitest.setup.ts +++ b/apps/sim/vitest.setup.ts @@ -1,23 +1,28 @@ -import { - authMock, - databaseMock, - drizzleOrmMock, - envFlagsMock, - environmentUtilsMock, - envMock, - hybridAuthMock, - loggerMock, - redisConfigMock, - requestUtilsMock, - schemaMock, - setupGlobalFetchMock, - setupGlobalStorageMocks, - terminalConsoleMock, - urlsMock, - workflowAuthzMock, -} from '@sim/testing' +import { authMock } from '@sim/testing/mocks/auth.mock' +import { databaseMock, drizzleOrmMock } from '@sim/testing/mocks/database.mock' +import { envMock } from '@sim/testing/mocks/env.mock' +import { envFlagsMock } from '@sim/testing/mocks/env-flags.mock' +import { environmentUtilsMock } from '@sim/testing/mocks/environment-utils.mock' +import { setupGlobalFetchMock } from '@sim/testing/mocks/fetch.mock' +import { hybridAuthMock } from '@sim/testing/mocks/hybrid-auth.mock' +import { loggerMock } from '@sim/testing/mocks/logger.mock' +import { redisConfigMock } from '@sim/testing/mocks/redis-config.mock' +import { requestUtilsMock } from '@sim/testing/mocks/request.mock' +import { schemaMock } from '@sim/testing/mocks/schema.mock' +import { setupGlobalStorageMocks } from '@sim/testing/mocks/storage.mock' +import { terminalConsoleMock } from '@sim/testing/mocks/terminal-console.mock' +import { urlsMock } from '@sim/testing/mocks/urls.mock' +import { workflowAuthzMock } from '@sim/testing/mocks/workflow-authz.mock' import { afterAll, vi } from 'vitest' +/** + * This file runs once per test file, and with `isolate: true` every module it + * touches is re-evaluated each time. The `@sim/testing` barrel is 69 modules + * (factories, builders, assertions, every mock); the 15 mocks registered here + * are 16. Importing them by file keeps the fixed per-file setup cost at + * ~10ms instead of ~300ms — measured on the full suite: setup 926s -> 30s. + */ + /** * jest-dom only registers DOM matchers (`toBeVisible`, `toHaveTextContent`, …), * so it is dead weight in a `node` environment — which is 985 of the 1,219 test @@ -140,6 +145,53 @@ vi.mock('@/blocks/registry', () => ({ ), })) +/** + * `@trigger.dev/core/v3` is ~350ms of externals per test file and reaches the + * route builders through `lib/core/async-jobs`. Only `taskContext.isInsideTask` + * is read at import time; the two suites that exercise it mock it themselves. + */ +vi.mock('@trigger.dev/core/v3', () => ({ taskContext: { isInsideTask: false } })) + +/** + * The generated tool metadata and output catalogs are ~5MB modules each. Tests + * that assert real tool params or outputs opt out with `vi.unmock(...)`. + */ +vi.mock('@/tools/metadata', () => ({ + getToolMetadata: vi.fn(() => undefined), + getToolParams: vi.fn(() => undefined), +})) +vi.mock('@/tools/metadata-outputs', () => ({ + getToolOutputsMetadata: vi.fn(() => undefined), +})) + +/** + * `@/components/icons` is a 10k-line sheet of brand SVGs that every block and + * trigger definition imports. Each name resolves to a stable stub component so + * identity comparisons and rendering both still work; tests that inspect real + * SVG markup opt out with `vi.unmock('@/components/icons')`. + */ +vi.mock('@/components/icons', async () => { + const React = await import('react') + const stubs = new Map>>() + const stubFor = (name: string) => { + let stub = stubs.get(name) + if (!stub) { + stub = (props) => React.createElement('svg', { 'data-icon': name, ...props }) + stub.displayName = name + stubs.set(name, stub) + } + return stub + } + return new Proxy( + {}, + { + get: (_target, name) => + typeof name === 'string' && name !== 'then' ? stubFor(name) : undefined, + has: (_target, name) => typeof name === 'string' && name !== 'then', + } + ) +}) + vi.mock('@trigger.dev/sdk', () => ({ task: vi.fn(() => ({ trigger: vi.fn() })), timeout: { None: 'none' }, diff --git a/package.json b/package.json index 84d1a3930c5..c724908ff7d 100644 --- a/package.json +++ b/package.json @@ -14,19 +14,8 @@ "dev:sockets": "cd apps/realtime && bun run dev", "dev:full": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"cd apps/sim && bun run dev\" \"cd apps/realtime && bun run dev\"", "dev:full:capped": "bunx concurrently -n \"App,Realtime\" -c \"cyan,magenta\" \"cd apps/sim && bun run dev:capped\" \"cd apps/realtime && bun run dev\"", - "test": "bun run test:setup && bun run test:npm-package-versions && bun run test:audit-candidates && bun run test:icon-path-precision && bun run test:tool-registry-boundary && bun run test:tool-request-boundary && bun run test:actorless-executor-operations && bun run test:permission-group-enforcement && bun run test:capability-subject && bun run test:application-graph && bun run test:migrations-safety && bun run test:generators && turbo run test", + "test": "bun run test:setup && bun run test:scripts && turbo run test", "test:setup": "bun run --cwd packages/sim-setup test", - "test:npm-package-versions": "bunx vitest run scripts/bump-npm-package-versions.test.ts", - "test:audit-candidates": "bunx vitest run scripts/check-db-audit-candidates.test.ts scripts/check-egress-boundary.test.ts", - "test:icon-path-precision": "bunx vitest run scripts/check-icon-path-precision.test.ts", - "test:tool-registry-boundary": "bunx vitest run scripts/check-tool-registry-boundary.test.ts", - "test:tool-request-boundary": "bunx vitest run scripts/check-tool-request-boundary.test.ts", - "test:actorless-executor-operations": "bunx vitest run scripts/check-actorless-executor-operations.test.ts", - "test:permission-group-enforcement": "bunx vitest run scripts/check-permission-group-enforcement.test.ts", - "test:capability-subject": "bunx vitest run scripts/check-capability-subject.test.ts", - "test:application-graph": "bunx vitest run scripts/check-application-graph.test.ts", - "test:migrations-safety": "bunx vitest run scripts/check-migrations-safety.test.ts", - "test:generators": "bunx vitest run scripts/generate-v2-cli-api.test.ts scripts/generate-cli-docs.test.ts scripts/generate-docs.test.ts scripts/generate-block-successors.test.ts", "format": "turbo run format", "format:check": "turbo run format:check", "lint": "turbo run lint", @@ -119,7 +108,8 @@ "agent-stream-docs:check": "bun run scripts/sync-agent-stream-docs.ts --check", "prepare": "bun husky", "type-check": "turbo run type-check", - "release": "bun run scripts/create-single-release.ts" + "release": "bun run scripts/create-single-release.ts", + "test:scripts": "vitest run --config vitest.scripts.config.ts" }, "overrides": { "react": "19.2.4", diff --git a/packages/testing/package.json b/packages/testing/package.json index b1ca9ac8b32..502fdf79a68 100644 --- a/packages/testing/package.json +++ b/packages/testing/package.json @@ -32,6 +32,10 @@ "types": "./src/mocks/executor.mock.ts", "default": "./src/mocks/executor.mock.ts" }, + "./mocks/*": { + "types": "./src/mocks/*.ts", + "default": "./src/mocks/*.ts" + }, "./assertions": { "types": "./src/assertions/index.ts", "default": "./src/assertions/index.ts" diff --git a/packages/testing/src/factories/tool-responses.factory.ts b/packages/testing/src/factories/tool-responses.factory.ts index a41e6575f88..3bf28a920dc 100644 --- a/packages/testing/src/factories/tool-responses.factory.ts +++ b/packages/testing/src/factories/tool-responses.factory.ts @@ -4,8 +4,6 @@ * This file contains mock data samples to be used in tool unit tests. */ -import { randomFloat } from '@sim/utils/random' - /** * HTTP Request mock responses for different scenarios. */ @@ -147,9 +145,7 @@ export const mockSheetsResponses = { */ export const mockPineconeResponses = { embedding: { - embedding: Array(1536) - .fill(0) - .map(() => randomFloat() * 2 - 1), + embedding: Array.from({ length: 1536 }, (_, i) => ((i * 7919) % 2000) / 1000 - 1), metadata: { text: 'Sample text for embedding', id: 'embed-123' }, }, searchResults: { diff --git a/packages/testing/src/mocks/tool-registry.mock.ts b/packages/testing/src/mocks/tool-registry.mock.ts new file mode 100644 index 00000000000..eb9c02f47e2 --- /dev/null +++ b/packages/testing/src/mocks/tool-registry.mock.ts @@ -0,0 +1,38 @@ +/** + * Builds a `@/tools/registry`-shaped map from one service's tool modules. + * + * The real registry is ~4,400 tools across ~6,000 modules — 20s+ of imports + * for every test file that unmocks it. A test that only needs its own + * service's configs registers a partial one instead: + * + * @example + * ```ts + * vi.mock('@/tools/registry', async () => { + * const { partialToolRegistry } = await import('@sim/testing/mocks/tool-registry.mock') + * return { tools: partialToolRegistry(await import('@/tools/pitchbook')) } + * }) + * ``` + * + * Registration itself is asserted with `hasToolId` from `@/tools/tool-ids`, + * which is generated from the registry and kept in sync by `tool-metadata:check`. + */ +export function partialToolRegistry( + ...modules: Array> +): Record { + const tools: Record = {} + for (const mod of modules) { + for (const value of Object.values(mod)) { + if (isToolLike(value)) tools[value.id] = value as T + } + } + return tools +} + +function isToolLike(value: unknown): value is { id: string } { + return ( + typeof value === 'object' && + value !== null && + typeof (value as { id?: unknown }).id === 'string' && + 'params' in value + ) +} diff --git a/scripts/check-script-test-coverage.ts b/scripts/check-script-test-coverage.ts index 3e126d5b164..783dce1bcdb 100644 --- a/scripts/check-script-test-coverage.ts +++ b/scripts/check-script-test-coverage.ts @@ -1,11 +1,14 @@ #!/usr/bin/env bun /** - * Asserts every `scripts/*.test.ts` file is reachable from the root `test` script. + * Asserts every `scripts/*.test.ts` file is collected by the root Vitest config. * - * The root `test` script chains a hand-maintained list of `test:*` entries, and a hand-maintained - * list silently drifts from the files on disk: a test added without a matching entry never runs, - * in CI or locally, and nothing reports it. `scripts/check-migrations-safety.test.ts` sat - * unreferenced and green for exactly that reason. + * The root `test` script once chained a hand-maintained list of `test:*` entries, and a + * hand-maintained list silently drifts from the files on disk: a test added without a matching + * entry never runs, in CI or locally, and nothing reports it. `scripts/check-migrations-safety.test.ts` + * sat unreferenced and green for exactly that reason. The root `vitest.scripts.config.ts` now collects + * the directory by glob, so drift can only come from a file the glob does not match (a test in a + * subdirectory, a different suffix) or from the `test` script no longer chaining `test:scripts`. + * This guard checks both by asking Vitest which files it would run. * * `run-audits.ts` derives its own list from the `check:*` namespace precisely so a new audit is * picked up by default, so this guard registers itself simply by being named `check:*` — it cannot @@ -15,57 +18,59 @@ import { readdirSync } from 'node:fs' import path from 'node:path' const ROOT = path.resolve(import.meta.dir, '..') -const TEST_FILE_PATTERN = /scripts\/[\w.-]+\.test\.ts/g const SUB_SCRIPT_PATTERN = /bun run ([\w:-]+)/g const manifest = await Bun.file(path.join(ROOT, 'package.json')).json() const commands = manifest.scripts as Record -/** Walks the `test` script and every `test:*` entry it chains, collecting referenced test files. */ -function reachableTestFiles(entry: string): Set { - const referenced = new Set() +/** Walks the `test` script and every entry it chains. */ +function reachableScripts(entry: string): Set { const seen = new Set() const queue = [entry] - while (queue.length > 0) { const name = queue.pop() as string if (seen.has(name)) continue seen.add(name) + for (const match of (commands[name] ?? '').matchAll(SUB_SCRIPT_PATTERN)) queue.push(match[1]) + } + return seen +} - const command = commands[name] - if (command === undefined) continue +if (!reachableScripts('test').has('test:scripts')) { + console.error('The root `test` script no longer chains `test:scripts`, so no script test runs.') + process.exit(1) +} - for (const match of command.matchAll(TEST_FILE_PATTERN)) referenced.add(match[0]) - for (const match of command.matchAll(SUB_SCRIPT_PATTERN)) queue.push(match[1]) +const listed = Bun.spawnSync( + ['bunx', 'vitest', 'list', '--json', '--config', 'vitest.scripts.config.ts'], + { + cwd: ROOT, } - - return referenced +) +if (listed.exitCode !== 0) { + console.error(`\`vitest list\` failed:\n${listed.stderr.toString()}`) + process.exit(1) } +const collected = new Set( + (JSON.parse(listed.stdout.toString()) as Array<{ file: string }>).map((entry) => + path.relative(ROOT, entry.file) + ) +) const onDisk = readdirSync(path.join(ROOT, 'scripts')) .filter((file) => file.endsWith('.test.ts')) .map((file) => `scripts/${file}`) .sort() -const reachable = reachableTestFiles('test') -const orphaned = onDisk.filter((file) => !reachable.has(file)) -const missing = [...reachable].filter((file) => !onDisk.includes(file)).sort() - -if (orphaned.length > 0 || missing.length > 0) { - if (orphaned.length > 0) { - console.error( - `Script tests never run by \`bun run test\`:\n${orphaned.map((file) => ` - ${file}`).join('\n')}\n` + - 'Add a `test:*` entry for each and chain it into the root `test` script.' - ) - } - if (missing.length > 0) { - console.error( - `Root \`test\` script references script tests that do not exist:\n${missing.map((file) => ` - ${file}`).join('\n')}` - ) - } +const orphaned = onDisk.filter((file) => !collected.has(file)) +if (orphaned.length > 0) { + console.error( + `Script tests never run by \`bun run test\`:\n${orphaned.map((file) => ` - ${file}`).join('\n')}\n` + + 'Make sure the root `vitest.scripts.config.ts` include glob matches them.' + ) process.exit(1) } console.log( - `Script test coverage passed: ${onDisk.length} script tests reachable from \`bun run test\`.` + `Script test coverage passed: ${onDisk.length} script tests collected by the root Vitest config.` ) diff --git a/turbo.json b/turbo.json index 619997da6ee..587768aa8e9 100644 --- a/turbo.json +++ b/turbo.json @@ -37,6 +37,7 @@ }, "test": { "dependsOn": ["^build"], + "env": ["SIM_TEST_SHARD"], "outputs": [] }, "lint": { diff --git a/vitest.scripts.config.ts b/vitest.scripts.config.ts new file mode 100644 index 00000000000..80d14604d7b --- /dev/null +++ b/vitest.scripts.config.ts @@ -0,0 +1,18 @@ +import { defineConfig } from 'vitest/config' + +/** + * Repo-level scripts have their own suites. One invocation for all of them + * replaces eleven sequential `vitest run ` processes, each of which paid + * its own startup. `scripts/openapi` keeps its own config and runs under + * `check:openapi`. + * + * Deliberately not named `vitest.config.ts`: Vitest walks up from a package's + * directory looking for that name, so a root config would silently replace + * the defaults of every workspace package that has none of its own. + */ +export default defineConfig({ + test: { + environment: 'node', + include: ['scripts/*.test.ts'], + }, +}) From 4658b636cd9c3698a11476078414aebbeda344f7 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 1 Sep 2026 12:53:55 -0700 Subject: [PATCH 2/2] fix(tests): recheck abort after lazy handler load; list script tests by file --- apps/sim/lib/copilot/request/tools/executor.ts | 3 ++- scripts/check-script-test-coverage.ts | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/copilot/request/tools/executor.ts b/apps/sim/lib/copilot/request/tools/executor.ts index dc39782d37d..d3a44749b6f 100644 --- a/apps/sim/lib/copilot/request/tools/executor.ts +++ b/apps/sim/lib/copilot/request/tools/executor.ts @@ -505,6 +505,8 @@ async function executeToolAndReportInner( }) } + // Loads the handler map on first use; the abort check below covers that wait. + await ensureHandlersRegistered() if (abortRequested(context, execContext, options)) { markToolCallCancelled('Request aborted before tool execution') markToolResultSeen(toolCall.id) @@ -608,7 +610,6 @@ async function executeToolAndReportInner( } try { - await ensureHandlersRegistered() let result = await executeToolWithWatchdog(toolCall, toolExecutionContext) if (toolCall.endTime || isTerminalToolCallStatus(toolCall.status)) { endToolSpanFromTerminalState() diff --git a/scripts/check-script-test-coverage.ts b/scripts/check-script-test-coverage.ts index 783dce1bcdb..87dc148c458 100644 --- a/scripts/check-script-test-coverage.ts +++ b/scripts/check-script-test-coverage.ts @@ -42,7 +42,7 @@ if (!reachableScripts('test').has('test:scripts')) { } const listed = Bun.spawnSync( - ['bunx', 'vitest', 'list', '--json', '--config', 'vitest.scripts.config.ts'], + ['bunx', 'vitest', 'list', '--json', '--filesOnly', '--config', 'vitest.scripts.config.ts'], { cwd: ROOT, } @@ -53,7 +53,7 @@ if (listed.exitCode !== 0) { } const collected = new Set( (JSON.parse(listed.stdout.toString()) as Array<{ file: string }>).map((entry) => - path.relative(ROOT, entry.file) + path.relative(ROOT, entry.file).split(path.sep).join('/') ) )